Improve speed performance
This commit is contained in:
Thibaut Dabonneville
parent
450d79b634
commit
aabd51116a
1 changed files with 158 additions and 142 deletions
|
|
@ -161,6 +161,9 @@ EXIT STATUS
|
||||||
4
|
4
|
||||||
Error during invocation of the command.
|
Error during invocation of the command.
|
||||||
|
|
||||||
|
5
|
||||||
|
The parent Asset Id doesn't exists.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
STOP
|
STOP
|
||||||
|
|
@ -176,15 +179,23 @@ if (!($^O =~ /^Win/i) && $> != 0 && !$override) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
my %ListAssetExists;
|
||||||
my %filelisthash;
|
my %filelisthash;
|
||||||
print "Starting..." unless ($quiet);
|
print "Starting..." unless ($quiet);
|
||||||
WebGUI::Session::open($webguiRoot,$configFile);
|
WebGUI::Session::open($webguiRoot,$configFile);
|
||||||
WebGUI::Session::refreshUserInfo(3);
|
WebGUI::Session::refreshUserInfo(3);
|
||||||
print "OK\n" unless ($quiet);
|
print "OK\n" unless ($quiet);
|
||||||
|
my $parent = WebGUI::Asset::File->newByDynamicClass($parentAssetId);
|
||||||
|
if (defined $parent) {
|
||||||
|
&buildListAssetExists($parent);
|
||||||
|
} else {
|
||||||
|
print "Warning: Parent asset '".$parentAssetId."' does not exist. Cannot import files.\n";
|
||||||
|
exit 5;
|
||||||
|
}
|
||||||
|
print "End of the childs detection\n" unless ($quiet);
|
||||||
# TB : wrap buildFileList in buildFileListWrap function for recursive search.
|
# TB : wrap buildFileList in buildFileListWrap function for recursive search.
|
||||||
&addFiles(buildFileListWrap($pathToFiles));
|
&addFiles(&buildFileListWrap($pathToFiles));
|
||||||
setPrivileges();
|
&setPrivileges();
|
||||||
|
|
||||||
print "Cleaning up..." unless ($quiet);
|
print "Cleaning up..." unless ($quiet);
|
||||||
WebGUI::Session::end($session{var}{sessionId});
|
WebGUI::Session::end($session{var}{sessionId});
|
||||||
|
|
@ -196,41 +207,39 @@ exit 0;
|
||||||
# addFiles(dbHandler, filelistHashRef, webguiSettingsHashRef, pathToCopyFrom)
|
# addFiles(dbHandler, filelistHashRef, webguiSettingsHashRef, pathToCopyFrom)
|
||||||
#-----------------------------------------
|
#-----------------------------------------
|
||||||
sub addFiles {
|
sub addFiles {
|
||||||
my $filelist = shift;
|
my $filelist = shift;
|
||||||
print "Adding files...\n" unless ($quiet);
|
print "Adding files...\n" unless ($quiet);
|
||||||
my $parent = WebGUI::Asset::File->newByDynamicClass($parentAssetId);
|
foreach my $file (@{$filelist}) {
|
||||||
if (defined $parent) {
|
my $class = 'WebGUI::Asset::File';
|
||||||
foreach my $file (@{$filelist}) {
|
# TB : add the path relative to $pathToFile in the message.
|
||||||
my $class = 'WebGUI::Asset::File';
|
print "\tAdding ".$file->{relpath}.$slash.$file->{filename}." to the database.\n" unless ($quiet);
|
||||||
# TB : add the path relative to $pathToFile in the message.
|
my $templateId = 'PBtmpl0000000000000024';
|
||||||
print "\tAdding ".$file->{relpath}.$slash.$file->{filename}." to the database.\n" unless ($quiet);
|
if (isIn($file->{ext},@nailable)) {
|
||||||
my $templateId = 'PBtmpl0000000000000024';
|
$class = 'WebGUI::Asset::File::Image';
|
||||||
if (isIn($file->{ext},@nailable)) {
|
$templateId = 'PBtmpl0000000000000088'
|
||||||
$class = 'WebGUI::Asset::File::Image';
|
}
|
||||||
$templateId = 'PBtmpl0000000000000088'
|
my $url = $parent->getUrl.'/'.$file->{filename};
|
||||||
}
|
|
||||||
my $url = $parent->getUrl.'/'.$file->{filename};
|
|
||||||
|
|
||||||
# TB : Possibly detect if the Asset exists already.
|
# TB : Possibly detect if the Asset exists already.
|
||||||
my ($replaceAsset, $replaceAssetId, $child) = (0,"","");
|
my ($replaceAsset, $replaceAssetId, $child) = (0,"","");
|
||||||
($replaceAsset, $replaceAssetId, $child) = &checkAssetExists($parent,$url) if ($overwrite);
|
($replaceAsset, $replaceAssetId, $child) = &checkAssetExists($url) if ($overwrite);
|
||||||
|
|
||||||
if ($replaceAsset == 1) {
|
if ($replaceAsset == 1) {
|
||||||
# TB : If the Asset exists, just copy the file.
|
# TB : If the Asset exists, just copy the file.
|
||||||
# To be check.
|
# To be check.
|
||||||
my $storage = WebGUI::Storage->get($replaceAssetId);
|
my $storage = WebGUI::Storage->get($replaceAssetId);
|
||||||
print "\t\tAsset exists already. Replace the file.\n" unless ($quiet);
|
print "\t\tAsset exists already. Replace the file.\n" unless ($quiet);
|
||||||
my $filename = $storage->addFileFromFilesystem("$pathToFiles$slash$file->{relpath}$slash$file->{filename}");
|
my $filename = $storage->addFileFromFilesystem("$pathToFiles$slash$file->{relpath}$slash$file->{filename}");
|
||||||
$child->generateThumbnail if ($class eq 'WebGUI::Asset::File::Image');
|
$child->generateThumbnail if ($class eq 'WebGUI::Asset::File::Image');
|
||||||
$child->setSize($storage->getFileSize($filename));
|
$child->setSize($storage->getFileSize($filename));
|
||||||
} else {
|
} else {
|
||||||
print "\t\tCreate the new asset.\n" unless ($quiet);
|
print "\t\tCreate the new asset.\n" unless ($quiet);
|
||||||
my $storage = WebGUI::Storage->create;
|
my $storage = WebGUI::Storage->create;
|
||||||
my $filename = $storage->addFileFromFilesystem("$pathToFiles$slash$file->{relpath}$slash$file->{filename}");
|
my $filename = $storage->addFileFromFilesystem("$pathToFiles$slash$file->{relpath}$slash$file->{filename}");
|
||||||
# TB : possibly remove the extension if the ignoreExtInName feature enabled.
|
# TB : possibly remove the extension if the ignoreExtInName feature enabled.
|
||||||
my $filenameTitle = $filename;
|
my $filenameTitle = $filename;
|
||||||
$filenameTitle =~ s/\.$file->{ext}// if ($ignoreExtInName);
|
$filenameTitle =~ s/\.$file->{ext}// if ($ignoreExtInName);
|
||||||
my $newAsset = $parent->addChild({
|
my $newAsset = $parent->addChild({
|
||||||
className=>$class,
|
className=>$class,
|
||||||
title=>$filenameTitle,
|
title=>$filenameTitle,
|
||||||
menuTitle=>$filenameTitle,
|
menuTitle=>$filenameTitle,
|
||||||
|
|
@ -243,96 +252,93 @@ sub addFiles {
|
||||||
templateId=>$templateId,
|
templateId=>$templateId,
|
||||||
endDate=>32472169200,
|
endDate=>32472169200,
|
||||||
ownerUserId=>$owner
|
ownerUserId=>$owner
|
||||||
});
|
});
|
||||||
$newAsset->generateThumbnail if ($class eq 'WebGUI::Asset::File::Image');
|
$newAsset->generateThumbnail if ($class eq 'WebGUI::Asset::File::Image');
|
||||||
$newAsset->setSize($storage->getFileSize($filename));
|
$newAsset->setSize($storage->getFileSize($filename));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
print "Finished adding.\n" unless ($quiet);
|
||||||
print "Warning: Parent asset '".$parentAssetId."' does not exist. Cannot import files.\n";
|
|
||||||
}
|
|
||||||
print "Finished adding.\n" unless ($quiet);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#-----------------------------------------
|
#-----------------------------------------
|
||||||
# setPrivileges()
|
# setPrivileges()
|
||||||
#-----------------------------------------
|
#-----------------------------------------
|
||||||
sub setPrivileges {
|
sub setPrivileges {
|
||||||
print "Setting filesystem privileges.\n" unless ($quiet);
|
print "Setting filesystem privileges.\n" unless ($quiet);
|
||||||
if ($session{os}{type} = "Linuxish") {
|
if ($session{os}{type} = "Linuxish") {
|
||||||
unless (system("chown -R ".$webUser." ".$session{config}{uploadsPath})) {
|
unless (system("chown -R ".$webUser." ".$session{config}{uploadsPath})) {
|
||||||
print "Privileges set.\n" unless ($quiet);
|
print "Privileges set.\n" unless ($quiet);
|
||||||
} else {
|
} else {
|
||||||
print "Could not set privileges.\n";
|
print "Could not set privileges.\n";
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
print "Cannot set privileges on this platform.\n" unless ($quiet)
|
print "Cannot set privileges on this platform.\n" unless ($quiet)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#-----------------------------------------
|
#-----------------------------------------
|
||||||
# buildFileListWrap(pathToImportFiles)
|
# buildFileListWrap(pathToImportFiles)
|
||||||
#-----------------------------------------
|
#-----------------------------------------
|
||||||
sub buildFileListWrap {
|
sub buildFileListWrap {
|
||||||
my ($path) = @_;
|
my ($path) = @_;
|
||||||
my (@filelist);
|
my (@filelist);
|
||||||
print "Building file list.\n" unless ($quiet);
|
print "Building file list.\n" unless ($quiet);
|
||||||
@filelist = &buildFileList($now,$path);
|
@filelist = &buildFileList($now,$path);
|
||||||
# &filelistCheck(@filelist);
|
# &filelistCheck(@filelist);
|
||||||
print "File list complete.\n" unless ($quiet);
|
print "File list complete.\n" unless ($quiet);
|
||||||
return \@filelist;
|
return \@filelist;
|
||||||
}
|
}
|
||||||
|
|
||||||
#-----------------------------------------
|
#-----------------------------------------
|
||||||
# buildFileList(time,pathToImportFiles)
|
# buildFileList(time,pathToImportFiles)
|
||||||
#-----------------------------------------
|
#-----------------------------------------
|
||||||
sub buildFileList {
|
sub buildFileList {
|
||||||
my ($now,$path) = @_;
|
my ($now,$path) = @_;
|
||||||
my (@filelist, @files, $file, $filename, $ext);
|
my (@filelist, @files, $file, $filename, $ext);
|
||||||
if (opendir(FILES,$path)) {
|
if (opendir(FILES,$path)) {
|
||||||
@files = readdir(FILES);
|
@files = readdir(FILES);
|
||||||
foreach $file (@files) {
|
foreach $file (@files) {
|
||||||
next if ($file eq "." || $file eq "..");
|
next if ($file eq "." || $file eq "..");
|
||||||
my $fullpathfile = "$path$slash$file";
|
my $fullpathfile = "$path$slash$file";
|
||||||
if (-f "$fullpathfile") {
|
if (-f "$fullpathfile") {
|
||||||
$file =~ /(.*?)\.(.*?)$/;
|
$file =~ /(.*?)\.(.*?)$/;
|
||||||
$filename = $1;
|
$filename = $1;
|
||||||
$ext = $2;
|
$ext = $2;
|
||||||
# TB : filter process : skip files due to options : skipOlderThan and findByExt
|
# TB : filter process : skip files due to options : skipOlderThan and findByExt
|
||||||
next if (&skipFilter($fullpathfile,$ext,$now));
|
next if (&skipFilter($fullpathfile,$ext,$now));
|
||||||
# TB : due to recursive search. Not really nice.
|
# TB : due to recursive search. Not really nice.
|
||||||
my $relpath = &trailFile("$path");
|
my $relpath = &trailFile("$path");
|
||||||
# TB : check is the filelist doesn't contains two times the same file (file with the same name)
|
# TB : check is the filelist doesn't contains two times the same file (file with the same name)
|
||||||
# due to recursive call, this can happen.
|
# due to recursive call, this can happen.
|
||||||
if (exists $filelisthash{$file}) {
|
if (exists $filelisthash{$file}) {
|
||||||
print "Error: file \"$file\" exists at several locations.
|
print "Error: file \"$file\" exists at several locations.
|
||||||
Both \"$filelisthash{$file}\" and \"$relpath\" contains it.
|
Both \"$filelisthash{$file}\" and \"$relpath\" contains it.
|
||||||
Exit at the first error of this type.\n" unless ($quiet);
|
Exit at the first error of this type.\n" unless ($quiet);
|
||||||
exit 2;
|
exit 2;
|
||||||
}
|
|
||||||
push(@filelist,{ext=>$ext, filename=>$file, relpath=>$relpath});
|
|
||||||
$filelisthash{$file} = $relpath;
|
|
||||||
print "Found file $file.\n" unless ($quiet);
|
|
||||||
}
|
|
||||||
# TB : the recursive call
|
|
||||||
push(@filelist,&buildFileList($now,"$fullpathfile")) if ((-d "$fullpathfile") && $recursive);
|
|
||||||
}
|
|
||||||
closedir(FILES);
|
|
||||||
return @filelist;
|
|
||||||
} else {
|
|
||||||
print "Error: Could not open folder $path.\n" unless ($quiet);
|
|
||||||
exit 2;
|
|
||||||
}
|
}
|
||||||
|
push(@filelist,{ext=>$ext, filename=>$file, relpath=>$relpath});
|
||||||
|
$filelisthash{$file} = $relpath;
|
||||||
|
print "Found file $file.\n" unless ($quiet);
|
||||||
|
}
|
||||||
|
# TB : the recursive call
|
||||||
|
push(@filelist,&buildFileList($now,"$fullpathfile")) if ((-d "$fullpathfile") && $recursive);
|
||||||
|
}
|
||||||
|
closedir(FILES);
|
||||||
|
return @filelist;
|
||||||
|
} else {
|
||||||
|
print "Error: Could not open folder $path.\n" unless ($quiet);
|
||||||
|
exit 2;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#-----------------------------------------
|
#-----------------------------------------
|
||||||
# getType(filename)
|
# getType(filename)
|
||||||
#-----------------------------------------
|
#-----------------------------------------
|
||||||
sub getType {
|
sub getType {
|
||||||
my ($extension);
|
my ($extension);
|
||||||
$extension = $_[0];
|
$extension = $_[0];
|
||||||
$extension =~ s/.*\.(.*?)$/$1/;
|
$extension =~ s/.*\.(.*?)$/$1/;
|
||||||
return $extension;
|
return $extension;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -340,22 +346,22 @@ sub getType {
|
||||||
# skipFilter(file,ext,time)
|
# skipFilter(file,ext,time)
|
||||||
#-----------------------------------------
|
#-----------------------------------------
|
||||||
sub skipFilter {
|
sub skipFilter {
|
||||||
my ($file,$ext,$now) = @_;
|
my ($file,$ext,$now) = @_;
|
||||||
# TB : stat in Windows has a strange behaviour relativelly to Unix
|
# TB : stat in Windows has a strange behaviour relativelly to Unix
|
||||||
# the output of stat si an array of array.
|
# the output of stat si an array of array.
|
||||||
# to be check on Unix if this work correctly.
|
# to be check on Unix if this work correctly.
|
||||||
my @dev = stat "$file";
|
my @dev = stat "$file";
|
||||||
# TB : option skipOlderThan
|
# TB : option skipOlderThan
|
||||||
if ($now - $dev[0][9] > $skipOlderThan) {
|
if ($now - $dev[0][9] > $skipOlderThan) {
|
||||||
print "Found file $file.\n\tBut older than $skipOlderThan. Skip it.\n" unless ($quiet);
|
print "Found file $file.\n\tBut older than $skipOlderThan. Skip it.\n" unless ($quiet);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
# TB : option findByExt
|
# TB : option findByExt
|
||||||
if (($findByExt ne "") && ($findByExt !~ /(^|,)$ext(,|$)/)) {
|
if (($findByExt ne "") && ($findByExt !~ /(^|,)$ext(,|$)/)) {
|
||||||
print "Found file in $file.\n\tBut Extension doesn't match findByExt tag. Skip it.\n" unless ($quiet);
|
print "Found file in $file.\n\tBut Extension doesn't match findByExt tag. Skip it.\n" unless ($quiet);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -363,19 +369,19 @@ sub skipFilter {
|
||||||
# trailFile(path)
|
# trailFile(path)
|
||||||
#-----------------------------------------
|
#-----------------------------------------
|
||||||
sub trailFile {
|
sub trailFile {
|
||||||
# TB : this is ugly, but unable to find a better way to do it.
|
# TB : this is ugly, but unable to find a better way to do it.
|
||||||
# this is supposed to find the relative path to pathToFiles
|
# this is supposed to find the relative path to pathToFiles
|
||||||
my ($path) = @_;
|
my ($path) = @_;
|
||||||
my $pathToFiles_temp = $pathToFiles;
|
my $pathToFiles_temp = $pathToFiles;
|
||||||
if ($^O =~ /Win/) {
|
if ($^O =~ /Win/) {
|
||||||
$pathToFiles_temp =~ s/\\/\\\\/g;
|
$pathToFiles_temp =~ s/\\/\\\\/g;
|
||||||
} else {
|
} else {
|
||||||
$pathToFiles_temp =~ s/\//\\\//g;
|
$pathToFiles_temp =~ s/\//\\\//g;
|
||||||
}
|
}
|
||||||
$path =~ s/$pathToFiles_temp//;
|
$path =~ s/$pathToFiles_temp//;
|
||||||
$path =~ s/.//;
|
$path =~ s/.//;
|
||||||
$path = "." if ($path eq "");
|
$path = "." if ($path eq "");
|
||||||
return $path;
|
return $path;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -383,20 +389,30 @@ sub trailFile {
|
||||||
# checkAssetExists(self,url)
|
# checkAssetExists(self,url)
|
||||||
#-----------------------------------------
|
#-----------------------------------------
|
||||||
sub checkAssetExists {
|
sub checkAssetExists {
|
||||||
my ($parent,$url) = @_;
|
my ($url) = @_;
|
||||||
my $replaceAsset = 0;
|
my $replaceAsset = 0;
|
||||||
my $replaceAssetId = "";
|
my $replaceAssetId = "";
|
||||||
my $child;
|
my $child;
|
||||||
my @allelement = $parent->getLineage();
|
if (exists $ListAssetExists{$url}) {
|
||||||
foreach ( @{$allelement[0]} ) {
|
$child = $ListAssetExists{$url};
|
||||||
$child = WebGUI::Asset::File->newByDynamicClass($_);
|
$replaceAsset = 1;
|
||||||
if ((defined $child) && ($child->getUrl eq $url)) {
|
$replaceAssetId = $child->getId;
|
||||||
$replaceAsset = 1;
|
}
|
||||||
$replaceAssetId = $child->getId;
|
return($replaceAsset, $replaceAssetId, $child);
|
||||||
last;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return($replaceAsset, $replaceAssetId, $child);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
#-----------------------------------------
|
||||||
|
# buildListAssetExists(self,url)
|
||||||
|
#-----------------------------------------
|
||||||
|
sub buildListAssetExists {
|
||||||
|
my ($parent) = @_;
|
||||||
|
my @allelement = $parent->getLineage();
|
||||||
|
foreach my $elem ( @{$allelement[0]} ) {
|
||||||
|
my $child = WebGUI::Asset::File->newByDynamicClass($elem);
|
||||||
|
if (defined $child) {
|
||||||
|
$ListAssetExists{$child->getUrl} = $child;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue