From aabd51116a1094fa41d3b0939be4398df0293050 Mon Sep 17 00:00:00 2001 From: Thibaut Dabonneville Date: Mon, 6 Feb 2006 20:58:18 +0000 Subject: [PATCH] Improve speed performance --- sbin/fileImport.pl | 300 ++++++++++++++++++++++++--------------------- 1 file changed, 158 insertions(+), 142 deletions(-) diff --git a/sbin/fileImport.pl b/sbin/fileImport.pl index 6569df4a7..6b3573a54 100644 --- a/sbin/fileImport.pl +++ b/sbin/fileImport.pl @@ -161,6 +161,9 @@ EXIT STATUS 4 Error during invocation of the command. + 5 + The parent Asset Id doesn't exists. + STOP @@ -176,15 +179,23 @@ if (!($^O =~ /^Win/i) && $> != 0 && !$override) { } +my %ListAssetExists; my %filelisthash; print "Starting..." unless ($quiet); WebGUI::Session::open($webguiRoot,$configFile); WebGUI::Session::refreshUserInfo(3); print "OK\n" unless ($quiet); - +my $parent = WebGUI::Asset::File->newByDynamicClass($parentAssetId); +if (defined $parent) { + &buildListAssetExists($parent); +} else { + print "Warning: Parent asset '".$parentAssetId."' does not exist. Cannot import files.\n"; + exit 5; +} +print "End of the childs detection\n" unless ($quiet); # TB : wrap buildFileList in buildFileListWrap function for recursive search. -&addFiles(buildFileListWrap($pathToFiles)); -setPrivileges(); +&addFiles(&buildFileListWrap($pathToFiles)); +&setPrivileges(); print "Cleaning up..." unless ($quiet); WebGUI::Session::end($session{var}{sessionId}); @@ -196,41 +207,39 @@ exit 0; # addFiles(dbHandler, filelistHashRef, webguiSettingsHashRef, pathToCopyFrom) #----------------------------------------- sub addFiles { - my $filelist = shift; - print "Adding files...\n" unless ($quiet); - my $parent = WebGUI::Asset::File->newByDynamicClass($parentAssetId); - if (defined $parent) { - foreach my $file (@{$filelist}) { - my $class = 'WebGUI::Asset::File'; - # TB : add the path relative to $pathToFile in the message. - print "\tAdding ".$file->{relpath}.$slash.$file->{filename}." to the database.\n" unless ($quiet); - my $templateId = 'PBtmpl0000000000000024'; - if (isIn($file->{ext},@nailable)) { - $class = 'WebGUI::Asset::File::Image'; - $templateId = 'PBtmpl0000000000000088' - } - my $url = $parent->getUrl.'/'.$file->{filename}; + my $filelist = shift; + print "Adding files...\n" unless ($quiet); + foreach my $file (@{$filelist}) { + my $class = 'WebGUI::Asset::File'; + # TB : add the path relative to $pathToFile in the message. + print "\tAdding ".$file->{relpath}.$slash.$file->{filename}." to the database.\n" unless ($quiet); + my $templateId = 'PBtmpl0000000000000024'; + if (isIn($file->{ext},@nailable)) { + $class = 'WebGUI::Asset::File::Image'; + $templateId = 'PBtmpl0000000000000088' + } + my $url = $parent->getUrl.'/'.$file->{filename}; - # TB : Possibly detect if the Asset exists already. - my ($replaceAsset, $replaceAssetId, $child) = (0,"",""); - ($replaceAsset, $replaceAssetId, $child) = &checkAssetExists($parent,$url) if ($overwrite); + # TB : Possibly detect if the Asset exists already. + my ($replaceAsset, $replaceAssetId, $child) = (0,"",""); + ($replaceAsset, $replaceAssetId, $child) = &checkAssetExists($url) if ($overwrite); - if ($replaceAsset == 1) { - # TB : If the Asset exists, just copy the file. - # To be check. - my $storage = WebGUI::Storage->get($replaceAssetId); - print "\t\tAsset exists already. Replace the file.\n" unless ($quiet); - my $filename = $storage->addFileFromFilesystem("$pathToFiles$slash$file->{relpath}$slash$file->{filename}"); - $child->generateThumbnail if ($class eq 'WebGUI::Asset::File::Image'); - $child->setSize($storage->getFileSize($filename)); - } else { - print "\t\tCreate the new asset.\n" unless ($quiet); - my $storage = WebGUI::Storage->create; - my $filename = $storage->addFileFromFilesystem("$pathToFiles$slash$file->{relpath}$slash$file->{filename}"); - # TB : possibly remove the extension if the ignoreExtInName feature enabled. - my $filenameTitle = $filename; - $filenameTitle =~ s/\.$file->{ext}// if ($ignoreExtInName); - my $newAsset = $parent->addChild({ + if ($replaceAsset == 1) { + # TB : If the Asset exists, just copy the file. + # To be check. + my $storage = WebGUI::Storage->get($replaceAssetId); + print "\t\tAsset exists already. Replace the file.\n" unless ($quiet); + my $filename = $storage->addFileFromFilesystem("$pathToFiles$slash$file->{relpath}$slash$file->{filename}"); + $child->generateThumbnail if ($class eq 'WebGUI::Asset::File::Image'); + $child->setSize($storage->getFileSize($filename)); + } else { + print "\t\tCreate the new asset.\n" unless ($quiet); + my $storage = WebGUI::Storage->create; + my $filename = $storage->addFileFromFilesystem("$pathToFiles$slash$file->{relpath}$slash$file->{filename}"); + # TB : possibly remove the extension if the ignoreExtInName feature enabled. + my $filenameTitle = $filename; + $filenameTitle =~ s/\.$file->{ext}// if ($ignoreExtInName); + my $newAsset = $parent->addChild({ className=>$class, title=>$filenameTitle, menuTitle=>$filenameTitle, @@ -243,96 +252,93 @@ sub addFiles { templateId=>$templateId, endDate=>32472169200, ownerUserId=>$owner - }); - $newAsset->generateThumbnail if ($class eq 'WebGUI::Asset::File::Image'); - $newAsset->setSize($storage->getFileSize($filename)); - } - } - } else { - print "Warning: Parent asset '".$parentAssetId."' does not exist. Cannot import files.\n"; - } - print "Finished adding.\n" unless ($quiet); + }); + $newAsset->generateThumbnail if ($class eq 'WebGUI::Asset::File::Image'); + $newAsset->setSize($storage->getFileSize($filename)); + } + } + print "Finished adding.\n" unless ($quiet); } #----------------------------------------- # setPrivileges() #----------------------------------------- sub setPrivileges { - print "Setting filesystem privileges.\n" unless ($quiet); - if ($session{os}{type} = "Linuxish") { - unless (system("chown -R ".$webUser." ".$session{config}{uploadsPath})) { - print "Privileges set.\n" unless ($quiet); - } else { - print "Could not set privileges.\n"; - } - } else { - print "Cannot set privileges on this platform.\n" unless ($quiet) - } + print "Setting filesystem privileges.\n" unless ($quiet); + if ($session{os}{type} = "Linuxish") { + unless (system("chown -R ".$webUser." ".$session{config}{uploadsPath})) { + print "Privileges set.\n" unless ($quiet); + } else { + print "Could not set privileges.\n"; + } + } else { + print "Cannot set privileges on this platform.\n" unless ($quiet) + } } #----------------------------------------- # buildFileListWrap(pathToImportFiles) #----------------------------------------- sub buildFileListWrap { - my ($path) = @_; - my (@filelist); - print "Building file list.\n" unless ($quiet); - @filelist = &buildFileList($now,$path); - # &filelistCheck(@filelist); - print "File list complete.\n" unless ($quiet); - return \@filelist; + my ($path) = @_; + my (@filelist); + print "Building file list.\n" unless ($quiet); + @filelist = &buildFileList($now,$path); + # &filelistCheck(@filelist); + print "File list complete.\n" unless ($quiet); + return \@filelist; } #----------------------------------------- # buildFileList(time,pathToImportFiles) #----------------------------------------- sub buildFileList { - my ($now,$path) = @_; - my (@filelist, @files, $file, $filename, $ext); - if (opendir(FILES,$path)) { - @files = readdir(FILES); - foreach $file (@files) { - next if ($file eq "." || $file eq ".."); - my $fullpathfile = "$path$slash$file"; - if (-f "$fullpathfile") { - $file =~ /(.*?)\.(.*?)$/; - $filename = $1; - $ext = $2; - # TB : filter process : skip files due to options : skipOlderThan and findByExt - next if (&skipFilter($fullpathfile,$ext,$now)); - # TB : due to recursive search. Not really nice. - my $relpath = &trailFile("$path"); - # TB : check is the filelist doesn't contains two times the same file (file with the same name) - # due to recursive call, this can happen. - if (exists $filelisthash{$file}) { - print "Error: file \"$file\" exists at several locations. + my ($now,$path) = @_; + my (@filelist, @files, $file, $filename, $ext); + if (opendir(FILES,$path)) { + @files = readdir(FILES); + foreach $file (@files) { + next if ($file eq "." || $file eq ".."); + my $fullpathfile = "$path$slash$file"; + if (-f "$fullpathfile") { + $file =~ /(.*?)\.(.*?)$/; + $filename = $1; + $ext = $2; + # TB : filter process : skip files due to options : skipOlderThan and findByExt + next if (&skipFilter($fullpathfile,$ext,$now)); + # TB : due to recursive search. Not really nice. + my $relpath = &trailFile("$path"); + # TB : check is the filelist doesn't contains two times the same file (file with the same name) + # due to recursive call, this can happen. + if (exists $filelisthash{$file}) { + print "Error: file \"$file\" exists at several locations. Both \"$filelisthash{$file}\" and \"$relpath\" contains it. Exit at the first error of this type.\n" unless ($quiet); - exit 2; - } - push(@filelist,{ext=>$ext, filename=>$file, relpath=>$relpath}); - $filelisthash{$file} = $relpath; - print "Found file $file.\n" unless ($quiet); - } - # TB : the recursive call - push(@filelist,&buildFileList($now,"$fullpathfile")) if ((-d "$fullpathfile") && $recursive); - } - closedir(FILES); - return @filelist; - } else { - print "Error: Could not open folder $path.\n" unless ($quiet); - exit 2; + exit 2; } + push(@filelist,{ext=>$ext, filename=>$file, relpath=>$relpath}); + $filelisthash{$file} = $relpath; + print "Found file $file.\n" unless ($quiet); + } + # TB : the recursive call + push(@filelist,&buildFileList($now,"$fullpathfile")) if ((-d "$fullpathfile") && $recursive); + } + closedir(FILES); + return @filelist; + } else { + print "Error: Could not open folder $path.\n" unless ($quiet); + exit 2; + } } #----------------------------------------- # getType(filename) #----------------------------------------- sub getType { - my ($extension); - $extension = $_[0]; - $extension =~ s/.*\.(.*?)$/$1/; - return $extension; + my ($extension); + $extension = $_[0]; + $extension =~ s/.*\.(.*?)$/$1/; + return $extension; } @@ -340,22 +346,22 @@ sub getType { # skipFilter(file,ext,time) #----------------------------------------- sub skipFilter { - my ($file,$ext,$now) = @_; - # TB : stat in Windows has a strange behaviour relativelly to Unix - # the output of stat si an array of array. - # to be check on Unix if this work correctly. - my @dev = stat "$file"; - # TB : option skipOlderThan - if ($now - $dev[0][9] > $skipOlderThan) { - print "Found file $file.\n\tBut older than $skipOlderThan. Skip it.\n" unless ($quiet); - return 1; - } - # TB : option findByExt - if (($findByExt ne "") && ($findByExt !~ /(^|,)$ext(,|$)/)) { - print "Found file in $file.\n\tBut Extension doesn't match findByExt tag. Skip it.\n" unless ($quiet); - return 1; - } - return 0; + my ($file,$ext,$now) = @_; + # TB : stat in Windows has a strange behaviour relativelly to Unix + # the output of stat si an array of array. + # to be check on Unix if this work correctly. + my @dev = stat "$file"; + # TB : option skipOlderThan + if ($now - $dev[0][9] > $skipOlderThan) { + print "Found file $file.\n\tBut older than $skipOlderThan. Skip it.\n" unless ($quiet); + return 1; + } + # TB : option findByExt + if (($findByExt ne "") && ($findByExt !~ /(^|,)$ext(,|$)/)) { + print "Found file in $file.\n\tBut Extension doesn't match findByExt tag. Skip it.\n" unless ($quiet); + return 1; + } + return 0; } @@ -363,19 +369,19 @@ sub skipFilter { # trailFile(path) #----------------------------------------- sub trailFile { - # TB : this is ugly, but unable to find a better way to do it. - # this is supposed to find the relative path to pathToFiles - my ($path) = @_; - my $pathToFiles_temp = $pathToFiles; - if ($^O =~ /Win/) { - $pathToFiles_temp =~ s/\\/\\\\/g; - } else { - $pathToFiles_temp =~ s/\//\\\//g; - } - $path =~ s/$pathToFiles_temp//; - $path =~ s/.//; - $path = "." if ($path eq ""); - return $path; + # TB : this is ugly, but unable to find a better way to do it. + # this is supposed to find the relative path to pathToFiles + my ($path) = @_; + my $pathToFiles_temp = $pathToFiles; + if ($^O =~ /Win/) { + $pathToFiles_temp =~ s/\\/\\\\/g; + } else { + $pathToFiles_temp =~ s/\//\\\//g; + } + $path =~ s/$pathToFiles_temp//; + $path =~ s/.//; + $path = "." if ($path eq ""); + return $path; } @@ -383,20 +389,30 @@ sub trailFile { # checkAssetExists(self,url) #----------------------------------------- sub checkAssetExists { - my ($parent,$url) = @_; - my $replaceAsset = 0; - my $replaceAssetId = ""; - my $child; - my @allelement = $parent->getLineage(); - foreach ( @{$allelement[0]} ) { - $child = WebGUI::Asset::File->newByDynamicClass($_); - if ((defined $child) && ($child->getUrl eq $url)) { - $replaceAsset = 1; - $replaceAssetId = $child->getId; - last; - } - } - return($replaceAsset, $replaceAssetId, $child); + my ($url) = @_; + my $replaceAsset = 0; + my $replaceAssetId = ""; + my $child; + if (exists $ListAssetExists{$url}) { + $child = $ListAssetExists{$url}; + $replaceAsset = 1; + $replaceAssetId = $child->getId; + } + return($replaceAsset, $replaceAssetId, $child); } +#----------------------------------------- +# buildListAssetExists(self,url) +#----------------------------------------- +sub buildListAssetExists { + my ($parent) = @_; + my @allelement = $parent->getLineage(); + foreach my $elem ( @{$allelement[0]} ) { + my $child = WebGUI::Asset::File->newByDynamicClass($elem); + if (defined $child) { + $ListAssetExists{$child->getUrl} = $child; + } + } +} +