migrate import and export products to Asset Helpers

This commit is contained in:
Doug Bell 2011-03-29 23:25:56 -05:00
parent 380b9c7540
commit 01844b6fa2
3 changed files with 381 additions and 283 deletions

View file

@ -32,200 +32,31 @@ property templateId => (
namespace => "Shelf",
hoverHelp => ['shelf template help', 'Asset_Shelf'],
label => ['shelf template', 'Asset_Shelf'],
);
);
#-------------------------------------------------------------------
#----------------------------------------------------------------------------
=head2 exportProducts ( )
=head2 getHelpers ( )
Export all products from the WebGUI system in a CSV file. For details
about the file format, see importProducts.
Returns a temporary WebGUI::Storage object containing the file. The
file will be named siteProductData.csv.
Add the importCSV and exportCSV helpers to the Shelf
=cut
sub exportProducts {
my $self = shift;
my $session = $self->session;
my @columns = qw{varSku shortdescription price weight quantity};
my $productData = WebGUI::Text::joinCSV(qw{mastersku title}, @columns) . "\n";
@columns = map { $_ eq 'shortdescription' ? 'shortdesc' : $_ } @columns;
my $getAProduct = WebGUI::Asset::Sku::Product->getIsa($session);
while (my $product = $getAProduct->()) {
my $mastersku = $product->sku;
my $title = $product->getTitle;
my $collateri = $product->getAllCollateral('variantsJSON');
foreach my $collateral (@{ $collateri }) {
my @productFields = @{ $collateral }{ @columns };
$productData .= WebGUI::Text::joinCSV($mastersku, $title, @productFields);
$productData .= "\n";
}
}
my $storage = WebGUI::Storage->createTemp($session);
$storage->addFileFromScalar('siteProductData.csv', $productData);
return $storage;
}
override getHelpers => sub {
my ( $self ) = @_;
my $helpers = super();
#-------------------------------------------------------------------
=head2 importProducts ( $filePath )
Import products into the WebGUI system. If the master sku of a product
exists in the system, it will be updated. If master skus do not exist,
they will be added.
The first line of the file should contain only the name of the columns,
in any order. It may not contain comments.
These are the column names, each is required:
=over 4
=item *
mastersku
=item *
varsku
=item *
title
=item *
shortdescription
=item *
price
=item *
weight
=item *
quantity
=back
The following lines will contain product information. Blank
lines and anything following a '#' sign will be ignored from
the second line of the file, on to the end.
Returns 1 if the import has taken place. This is to help you know
if old data has been deleted and new has been inserted.
=cut
sub importProducts {
my $self = shift;
my $filePath = shift;
my $session = $self->session;
WebGUI::Error::InvalidParam->throw(error => q{Must provide the path to a file})
unless $filePath;
WebGUI::Error::InvalidFile->throw(error => qq{File could not be found}, brokenFile => $filePath)
unless -e $filePath;
WebGUI::Error::InvalidFile->throw(error => qq{File is not readable}, brokenFile => $filePath)
unless -r $filePath;
open my $table, '<', $filePath or
WebGUI::Error->throw(error => qq{Unable to open $filePath for reading: $!\n});
my $headers;
$headers = <$table>;
chomp $headers;
$headers =~ tr/\r//d;
$headers =~ s/\bsku\b/varSku/;
my @headers = WebGUI::Text::splitCSV($headers);
WebGUI::Error::InvalidFile->throw(error => qq{Bad header found in the CSV file}, brokenFile => $filePath)
unless (join(q{-}, sort @headers) eq 'mastersku-price-quantity-shortdescription-title-varSku-weight')
and (scalar @headers == 7);
my @productData = ();
my $line = 1;
while (my $productRow = <$table>) {
chomp $productRow;
$productRow =~ tr/\r//d;
$productRow =~ s/\s*#.+$//;
next unless $productRow;
local $_;
my @productRow = WebGUI::Text::splitCSV($productRow);
WebGUI::Error::InvalidFile->throw(error => qq{Error found in the CSV file}, brokenFile => $filePath, brokenLine => $line)
unless scalar @productRow == 7;
push @productData, [ @productRow ];
}
return unless scalar @productData;
##Okay, if we got this far, then the data looks fine.
my $fetchProductId = $session->db->prepare('select p.assetId from Product as p join sku as s on p.assetId=s.assetId and p.revisionDate=s.revisionDate where s.sku=? order by p.revisionDate DESC limit 1');
my $node = $self;
@headers = map { $_ eq 'shortdescription' ? 'shortdesc' : $_ } @headers;
my @collateralFields = grep { $_ ne 'title' and $_ ne 'mastersku' } @headers;
PRODUCT: foreach my $productRow (@productData) {
my %productRow;
##Order the data according to the headers, in whatever order they exist.
@productRow{ @headers } = @{ $productRow };
##Isolate just the collateral from the other product information
my %productCollateral;
@productCollateral{ @collateralFields } = @productRow{ @collateralFields };
$fetchProductId->execute([$productRow{mastersku}]);
my $asset = $fetchProductId->hashRef;
##If the assetId exists, we update data for it
if ($asset->{assetId}) {
$session->log->warn("Modifying an existing product: $productRow{sku} = $asset->{assetId}\n");
my $assetId = $asset->{assetId};
my $product = WebGUI::Asset->newPending($session, $assetId);
##Error handling for locked assets
if ($product->isLocked) {
$session->log->warn("Product is locked");
next PRODUCT if $product->isLocked;
}
if ($productRow{title} ne $product->getTitle) {
$product->update({
title => $productRow{title},
menuTitle => $productRow{title},
});
}
my $collaterals = $product->getAllCollateral('variantsJSON');
my $collateralSet = 0;
ROW: foreach my $collateral (@{ $collaterals }) {
next ROW unless $collateral->{varSku} eq $productRow{varSku};
@{ $collateral}{ @collateralFields } = @productCollateral{ @collateralFields }; ##preserve the variant Id field, assign all others
$product->setCollateral('variantsJSON', 'variantId', $collateral->{variantId}, $collateral);
$collateralSet=1;
}
if (!$collateralSet) {
##It must be a new variant
$product->setCollateral('variantsJSON', 'variantId', 'new', \%productCollateral);
}
}
else {
##Insert a new product;
$session->log->warn("Making a new product: $productRow{sku}\n");
my $newProduct = $node->addChild({className => 'WebGUI::Asset::Sku::Product'});
$newProduct->update({
title => $productRow{title},
menuTitle => $productRow{title},
url => $productRow{title},
sku => $productRow{mastersku},
});
$newProduct->setCollateral('variantsJSON', 'variantId', 'new', \%productCollateral);
$newProduct->commit;
}
}
return 1;
}
$helpers->{import_products} = {
className => 'WebGUI::AssetHelper::Product::ImportCSV',
label => 'Import Products',
};
$helpers->{export_products} = {
className => 'WebGUI::AssetHelper::Product::ExportCSV',
label => 'Export Products',
};
return $helpers;
};
#-------------------------------------------------------------------
@ -331,102 +162,5 @@ sub view {
return $self->processTemplate(\%var, undef, $self->{_viewTemplate});
}
#-------------------------------------------------------------------
=head2 www_edit ( )
Override the superclass to add import and exprt items to the AdminConsole submenu.
=cut
override www_edit => sub {
my $self = shift;
my $i18n = WebGUI::International->new($self->session, 'Asset_Shelf');
if ($self->getId ne "new") {
$self->getAdminConsole->addSubmenuItem($self->getUrl('func=exportProducts'),$i18n->get("export"));
$self->getAdminConsole->addSubmenuItem($self->getUrl('func=importProducts'),$i18n->get("import"));
}
return super();
};
#-------------------------------------------------------------------
=head2 www_exportProducts ( )
Export all product SKUs as a CSV file. Returns a WebGUI::Storage
object containg the product file, named 'siteProductData.csv'.
=cut
sub www_exportProducts {
my $self = shift;
my $session = $self->session;
return $session->privilege->insufficient
unless $self->canEdit;
my $storage = $self->exportProducts();
$session->http->setRedirect($storage->getUrl($storage->getFiles->[0]));
return "redirect";
}
#-------------------------------------------------------------------
=head2 www_importProducts ( )
Import new product data from a file provided by the user. This will create new products
or alter existing products.
=cut
sub www_importProducts {
my $self = shift;
my $session = $self->session;
return $session->privilege->insufficient unless $self->canEdit;
my $i18n=WebGUI::International->new($session, 'Asset_Shelf');
my $status_message;
if ( $session->form->get('doit')) {
my $storage = WebGUI::Storage->create($session);
my $productFile = $storage->addFileFromFormPost('importFile', 1);
eval {
$self->importProducts($storage->getPath($productFile)) if $productFile;
};
my $exception;
if ($exception = Exception::Class->caught('WebGUI::Error::InvalidFile')) {
$status_message = sprintf 'A problem was found with your file: %s',
$exception->error;
if ($exception->brokenLine) {
$status_message .= sprintf ' on line %d', $exception->brokenLine;
}
}
elsif ($exception = Exception::Class->caught()) {
$status_message = sprintf 'A problem happened during the import: %s', $exception->error;
}
else {
$status_message = $i18n->get('import successful');
if (WebGUI::VersionTag->autoCommitWorkingIfEnabled($self->session, {
allowComments => 1,
returnUrl => $self->getUrl,
}) eq 'redirect') {
return undef;
};
}
}
my $output;
if ($status_message) {
$output = '<div id="status_message">'.$status_message.'</div>';
}
$output .= WebGUI::Form::formHeader($session,{action => $self->getUrl})
. WebGUI::Form::hidden($session, {name=>"func", value=>"importProducts"})
. WebGUI::Form::hidden($session, {name=>"doit", value=>1})
. q{<input type="file" name="importFile" size="10" />}
. WebGUI::Form::submit($session,{value=>$i18n->get('import'), extras=>q{style="float: left;"} })
. WebGUI::Form::formFooter($session);
return $self->getAdminConsole->render($output, $i18n->get('import'));
}
__PACKAGE__->meta->make_immutable;
1;

View file

@ -0,0 +1,95 @@
package WebGUI::AssetHelper::Product::ExportCSV;
use Moose;
extends 'WebGUI::AssetHelper';
use JSON;
use WebGUI::Asset::Sku::Product;
use WebGUI::Fork;
use WebGUI::Text;
use WebGUI::Asset;
use WebGUI::Storage;
#-------------------------------------------------------------------
=head2 process ( )
Fork the copy operation
=cut
sub process {
my ($self) = @_;
my $asset = $self->asset;
my $session = $self->session;
# Fork the export. Forking makes sure it won't get interrupted
my $fork = WebGUI::Fork->start(
$session, blessed( $self ), 'exportProducts',
);
return {
forkId => $fork->getId,
};
}
#-------------------------------------------------------------------
=head2 exportProducts ( )
Export all products from the WebGUI system in a CSV file. For details
about the file format, see WebGUI::AssetHelper::Product::ImportCSV
Returns a temporary WebGUI::Storage object containing the file. The
file will be named siteProductData.csv.
=cut
sub exportProducts {
my ($process, $args) = @_;
my $session = $process->session;
# Get all the product IDs
# Not using getIsa so I can have the number to put into the progress bar
# This should be perhaps genericized and placed into WebGUI::Asset
my $tableName = $session->db->dbh->quote_identifier( WebGUI::Asset::Sku::Product->tableName );
my $productIds = $session->db->buildArrayRef(
"SELECT assetId FROM asset JOIN assetData USING (assetId) JOIN $tableName USING (assetId, revisionDate) WHERE status=? OR status=? HAVING MAX(revisionDate)",
['approved','archived'],
);
# Preparing to dispense product
my $status = {
message => 'Dispensing product...',
total => scalar @{$productIds},
finished => 0,
};
$process->update( sub { JSON->new->encode( $status ) } );
# Dispensing product
my @columns = qw{varSku shortdescription price weight quantity};
my $productData = WebGUI::Text::joinCSV(qw{mastersku title}, @columns) . "\n";
@columns = map { $_ eq 'shortdescription' ? 'shortdesc' : $_ } @columns;
for my $productId ( @$productIds ) {
my $product = WebGUI::Asset->newById( $session, $productId );
my $mastersku = $product->sku;
my $title = $product->getTitle;
my $collateri = $product->getAllCollateral('variantsJSON');
foreach my $collateral (@{ $collateri }) {
my @productFields = @{ $collateral }{ @columns };
$productData .= WebGUI::Text::joinCSV($mastersku, $title, @productFields);
$productData .= "\n";
}
$status->{finished}++;
$process->update( sub { JSON->new->encode( $status ) } );
}
my $storage = WebGUI::Storage->createTemp($session);
$storage->addFileFromScalar('siteProductData.csv', $productData);
# Are you still there?
$status->{redirect} = $storage->getUrl( 'siteProductData.csv' );
$process->update( sub { JSON->new->encode( $status ) } );
$session->log->info( "Products exported to " . $status->{redirect} );
}
1;

View file

@ -0,0 +1,269 @@
package WebGUI::AssetHelper::Product::ImportCSV;
use Moose;
extends 'WebGUI::AssetHelper';
use PerlIO::eol;
use JSON;
use WebGUI::Exception;
use WebGUI::Fork;
use WebGUI::Text;
use WebGUI::Storage;
use WebGUI::International;
#-------------------------------------------------------------------
=head2 process ( )
Display a dialog to import products
=cut
sub process {
my ($self) = @_;
my $asset = $self->asset;
my $session = $self->session;
return {
openDialog => $self->getUrl( 'importProducts' ),
};
}
#-------------------------------------------------------------------
=head2 importProducts ( )
Import products into the WebGUI system. If the master sku of a product
exists in the system, it will be updated. If master skus do not exist,
they will be added.
The first line of the file should contain only the name of the columns,
in any order. It may not contain comments.
These are the column names, each is required:
=over 4
=item *
mastersku
=item *
varsku
=item *
title
=item *
shortdescription
=item *
price
=item *
weight
=item *
quantity
=back
The following lines will contain product information. Blank
lines and anything following a '#' sign will be ignored from
the second line of the file, on to the end.
Returns 1 if the import has taken place. This is to help you know
if old data has been deleted and new has been inserted.
=cut
sub importProducts {
my ( $process, $args ) = @_;
my $session = $process->session;
my $asset = WebGUI::Asset->newById( $session, $args->{assetId} );
my $filePath = $args->{filePath};
WebGUI::Error::InvalidParam->throw(error => q{Must provide the path to a file})
unless $filePath;
WebGUI::Error::InvalidFile->throw(error => qq{File could not be found}, brokenFile => $filePath)
unless -e $filePath;
WebGUI::Error::InvalidFile->throw(error => qq{File is not readable}, brokenFile => $filePath)
unless -r $filePath;
local $/ = "\x0A"; # Fork alters this!!!
open my $table, '<:raw:eol(CRLF)', $filePath or
WebGUI::Error->throw(error => qq{Unable to open $filePath for reading: $!\n});
# Read in the data
my $headers;
$headers = <$table>;
$session->log->info( "Headers: " . $headers );
chomp $headers;
$headers =~ tr/\r//d;
$headers =~ s/\bsku\b/varSku/;
my @headers = WebGUI::Text::splitCSV($headers);
unless ( (join(q{-}, sort @headers) eq 'mastersku-price-quantity-shortdescription-title-varSku-weight')
and (scalar @headers == 7) ) {
$session->log->error( "Bad header found in CSV file ($filePath): $headers -- " . join ", ", sort @headers );
WebGUI::Error::InvalidFile->throw(error => qq{Bad header found in the CSV file}, brokenFile => $filePath);
}
my @productData = ();
my $line = 1;
while (my $productRow = <$table>) {
$session->log->info( "Product: " . $productRow );
chomp $productRow;
$productRow =~ tr/\r//d;
$productRow =~ s/\s*#.+$//;
next unless $productRow;
local $_;
my @productRow = WebGUI::Text::splitCSV($productRow);
WebGUI::Error::InvalidFile->throw(error => qq{Error found in the CSV file}, brokenFile => $filePath, brokenLine => $line)
unless scalar @productRow == 7;
push @productData, [ @productRow ];
}
if ( @productData == 0 ) {
$session->log->warn("No products to import");
$process->update( sub { JSON->new->encode( { message => 'No products' } ) } );
$process->finish;
return;
}
# Preparing to load product
my $status = {
message => 'Loading product...',
total => scalar @productData,
finished => 0,
};
$process->update( sub { JSON->new->encode( $status ) } );
##Okay, if we got this far, then the data looks fine.
my $fetchProductId = $session->db->prepare('select p.assetId from Product as p join sku as s on p.assetId=s.assetId and p.revisionDate=s.revisionDate where s.sku=? order by p.revisionDate DESC limit 1');
@headers = map { $_ eq 'shortdescription' ? 'shortdesc' : $_ } @headers;
my @collateralFields = grep { $_ ne 'title' and $_ ne 'mastersku' } @headers;
PRODUCT: foreach my $productRow (@productData) {
my %productRow;
##Order the data according to the headers, in whatever order they exist.
@productRow{ @headers } = @{ $productRow };
##Isolate just the collateral from the other product information
my %productCollateral;
@productCollateral{ @collateralFields } = @productRow{ @collateralFields };
$fetchProductId->execute([$productRow{mastersku}]);
my $asset = $fetchProductId->hashRef;
##If the assetId exists, we update data for it
if ($asset->{assetId}) {
$session->log->warn("Modifying an existing product: $productRow{sku} = $asset->{assetId}\n");
my $assetId = $asset->{assetId};
my $product = WebGUI::Asset->newPending($session, $assetId);
##Error handling for locked assets
if ($product->isLocked) {
$session->log->warn("Product is locked");
next PRODUCT if $product->isLocked;
}
if ($productRow{title} ne $product->getTitle) {
$product->update({
title => $productRow{title},
menuTitle => $productRow{title},
});
}
my $collaterals = $product->getAllCollateral('variantsJSON');
my $collateralSet = 0;
ROW: foreach my $collateral (@{ $collaterals }) {
next ROW unless $collateral->{varSku} eq $productRow{varSku};
@{ $collateral}{ @collateralFields } = @productCollateral{ @collateralFields }; ##preserve the variant Id field, assign all others
$product->setCollateral('variantsJSON', 'variantId', $collateral->{variantId}, $collateral);
$collateralSet=1;
}
if (!$collateralSet) {
##It must be a new variant
$product->setCollateral('variantsJSON', 'variantId', 'new', \%productCollateral);
}
}
else {
##Insert a new product;
$session->log->warn("Making a new product: $productRow{sku}\n");
my $newProduct = $asset->addChild({className => 'WebGUI::Asset::Sku::Product'});
$newProduct->update({
title => $productRow{title},
menuTitle => $productRow{title},
url => $productRow{title},
sku => $productRow{mastersku},
});
$newProduct->setCollateral('variantsJSON', 'variantId', 'new', \%productCollateral);
$newProduct->commit;
}
# Update our status
$status->{finished}++;
$process->update( sub { JSON->new->encode( $status ) } );
}
$process->finish;
}
#-----------------------------------------------------------------------------
=head2 www_importProducts ( )
Show the form to upload the CSV file
=cut
sub www_importProducts {
my $self = shift;
my $session = $self->session;
return $session->privilege->insufficient unless $self->asset->canEdit;
my $i18n = WebGUI::International->new( $session, 'Asset_Shelf' );
my $f = $self->getForm( 'importProductsSave' );
$f->addField( 'file', name => 'importFile' );
$f->addField( 'submit', name => 'submit', value => $i18n->get('import') );
return $session->style->process(
'<h1>' . $i18n->get('import') . '</h1>' . $f->toHtml,
"PBtmplBlankStyle000001"
);
}
#-----------------------------------------------------------------------------
=head2 www_importProductsSave ( )
Import the products from the CSV file in a forked process
=cut
sub www_importProductsSave {
my ( $self ) = @_;
my $session = $self->session;
my $storage = WebGUI::Storage->create($session);
my $productFile = $storage->addFileFromFormPost( 'importFile_file', 1 );
# Fork the import
my $fork = WebGUI::Fork->start(
$session, blessed( $self ), 'importProducts',
{ assetId => $self->asset->getId, filePath => $storage->getPath( $productFile ), },
);
my $output = '<script type="text/javascript">'
. sprintf( 'window.parent.admin.processPlugin({ forkId : "%s" });', $fork->getId )
. 'window.parent.admin.closeModalDialog();'
. '</script>'
;
return $session->style->process(
$output,
"PBtmplBlankStyle000001"
);
}
1;