From 21573a54f11d0ff67e90a79ca7cf6a51e61a8cc1 Mon Sep 17 00:00:00 2001 From: Martin Kamerbeek Date: Mon, 26 Feb 2007 21:43:53 +0000 Subject: [PATCH] fileImport fixes. --- docs/changelog/7.x.x.txt | 4 +- sbin/fileImport.pl | 259 ++++++++++++++++----------------------- 2 files changed, 110 insertions(+), 153 deletions(-) diff --git a/docs/changelog/7.x.x.txt b/docs/changelog/7.x.x.txt index 1d506b04a..3dcd0cf52 100644 --- a/docs/changelog/7.x.x.txt +++ b/docs/changelog/7.x.x.txt @@ -6,7 +6,9 @@ - fix: Off by 1 error while displaying ad impressions (perlDreamer Consulting, LLC) - fix: Wrong link in template toolkit help (perlDreamer Consulting, LLC) - fix: Outdated info in Env macro help (perlDreamer Consulting, LLC) - + - fix: A bunch of bugs in fileImport.pl and improved its performance very + significantly (Martin Kamerbeek / Oqapi) + 7.3.10 - fix: Calendar Update Feeds now handles iCalendar "DURATION" field - fix: Various potential bugs in WebGUI::DateTime due to inheritence from diff --git a/sbin/fileImport.pl b/sbin/fileImport.pl index 44d017c35..1f179c13a 100644 --- a/sbin/fileImport.pl +++ b/sbin/fileImport.pl @@ -53,31 +53,31 @@ my $overwrite = ''; my $ignoreExtInName = ''; GetOptions( - 'configFile=s'=>\$configFile, - 'owner=s'=>\$owner, - 'groupToView=s'=>\$groupToView, - 'groupToEdit=s'=>\$groupToEdit, - 'help'=>\$help, - 'override'=>\$override, - 'pathToFiles=s'=>\$pathToFiles, - 'quiet'=>\$quiet, - 'webUser=s'=>\$webUser, - 'parentAssetId=s'=>\$parentAssetId, - 'skipOlderThan=i'=>\$skipOlderThan, - 'findByExt=s'=>\$findByExt, - 'recursive'=>\$recursive, - 'overwrite'=>\$overwrite, - 'ignoreExtInName'=>\$ignoreExtInName + 'configFile=s' => \$configFile, + 'owner=s' => \$owner, + 'groupToView=s' => \$groupToView, + 'groupToEdit=s' => \$groupToEdit, + 'help' => \$help, + 'override' => \$override, + 'pathToFiles=s' => \$pathToFiles, + 'quiet' => \$quiet, + 'webUser=s' => \$webUser, + 'parentAssetId=s' => \$parentAssetId, + 'skipOlderThan=i' => \$skipOlderThan, + 'findByExt=s' => \$findByExt, + 'recursive' => \$recursive, + 'overwrite' => \$overwrite, + 'ignoreExtInName' => \$ignoreExtInName ); if ($configFile eq "" || $pathToFiles eq "" || $parentAssetId eq "") { - &printHelp; + printHelp(); exit 4; }; if ($help) { - &printHelp; + printHelp(); exit 1; } @@ -185,17 +185,19 @@ $session->user({userId=>3}); print "OK\n" unless ($quiet); my $parent = WebGUI::Asset::File->newByDynamicClass($session, $parentAssetId); -if (defined $parent) { - &buildListAssetExists($parent); -} else { +unless (defined $parent) { print "Warning: Parent asset '".$parentAssetId."' does not exist. Cannot import files.\n"; exit 5; } print "End of the childs detection\n" unless ($quiet); # TB : wrap buildFileList in buildFileListWrap function for recursive search. -&addFiles(&buildFileListWrap($pathToFiles)); -&setPrivileges(); +addFiles(buildFileListWrap($pathToFiles)); + +print "Committing version tag..." unless ($quiet); +my $versionTag = WebGUI::VersionTag->getWorking($session); +$versionTag->commit; +print "Done.\n" unless ($quiet); print "Cleaning up..." unless ($quiet); $session->var->end(); @@ -208,74 +210,97 @@ exit 0; # addFiles(dbHandler, filelistHashRef, webguiSettingsHashRef, pathToCopyFrom) #----------------------------------------- sub addFiles { + my ($class, $templateId, $url, $newAsset); my $filelist = shift; + print "Adding files...\n" unless ($quiet); foreach my $file (@{$filelist}) { - my $class = 'WebGUI::Asset::File'; - - # TB : add the path relative to $pathToFile in the message. - print "\tAdding ".$file->{relpath}.$slash.$file->{filename}." to the database.\n" unless ($quiet); - - my $templateId = 'PBtmpl0000000000000024'; + print "\tAdding ".$file->{fullPathFile}." to the database.\n" unless ($quiet); + + # Figure out whether the file is an image or not by its extension. if (isIn(lc($file->{ext}),@nailable)) { $class = 'WebGUI::Asset::File::Image'; - $templateId = 'PBtmpl0000000000000088' + $templateId = 'PBtmpl0000000000000088'; + } + else { + $class = 'WebGUI::Asset::File'; + $templateId = 'PBtmpl0000000000000024'; } - my $url = $parent->getUrl.'/'.$file->{filename}; - # TB : Possibly detect if the Asset exists already. - my ($replaceAsset, $replaceAssetId, $child) = (0,"",""); - ($replaceAsset, $replaceAssetId, $child) = &checkAssetExists($url) if ($overwrite); + $url = $parent->getUrl.'/'.$file->{filename}; + $url =~ s{^/}{}; - if ($replaceAsset == 1) { - # TB : If the Asset exists, just copy the file. - # To be check. - my $storage = WebGUI::Storage->get($session, $replaceAssetId); + # Create a new storage location and add the file to it. + my $storage = WebGUI::Storage->create($session); + my $filename = $storage->addFileFromFilesystem($file->{fullPathFile}); + + # TB : possibly remove the extension if the ignoreExtInName feature enabled. + my $filenameTitle = $filename; + $filenameTitle =~ s/\.$file->{ext}// if ($ignoreExtInName); + + # Set up the properties of the file to be added. + my $assetProperties = { + className => $class, + title => $filenameTitle, + menuTitle => $filenameTitle, + filename => $filename, + storageId => $storage->getId, + isHidden => 1, + url => $url, + groupIdView => $groupToView, + groupIdEdit => $groupToEdit, + templateId => $templateId, + endDate => 32472169200, + ownerUserId => $owner, + }; + + if (WebGUI::Asset->urlExists($session, $url) && $overwrite) { print "\t\tAsset exists already. Replace the file.\n" unless ($quiet); - my $filename = $storage->addFileFromFilesystem("$pathToFiles$slash$file->{relpath}$slash$file->{filename}"); - $child->generateThumbnail if ($class eq 'WebGUI::Asset::File::Image'); - $child->setSize($storage->getFileSize($filename)); + + # Add a new revision. + my $originalAsset = WebGUI::Asset->newByUrl($session, $url); + $newAsset = $originalAsset->addRevision($assetProperties); } else { print "\t\tCreate the new asset.\n" unless ($quiet); - my $storage = WebGUI::Storage->create($session); - my $filename = $storage->addFileFromFilesystem("$pathToFiles$slash$file->{relpath}$slash$file->{filename}"); - - # TB : possibly remove the extension if the ignoreExtInName feature enabled. - my $filenameTitle = $filename; - $filenameTitle =~ s/\.$file->{ext}// if ($ignoreExtInName); - my $newAsset = $parent->addChild({ - className => $class, - title => $filenameTitle, - menuTitle => $filenameTitle, - filename => $filename, - storageId => $storage->getId, - isHidden => 1, - url => $url, - groupIdView => $groupToView, - groupIdEdit => $groupToEdit, - templateId => $templateId, - endDate => 32472169200, - ownerUserId => $owner, - }); - - $newAsset->generateThumbnail if ($class eq 'WebGUI::Asset::File::Image'); - $newAsset->setSize($storage->getFileSize($filename)); + # Add a new asset. + $newAsset = $parent->addChild($assetProperties); } + + # Create thumbnail, scale and set size. + if ($class eq 'WebGUI::Asset::File::Image') { + # Generate thumbnail + $newAsset->generateThumbnail($session->setting->get('thumbnailSize')); + + # Resize image to maxImageSize if necessary. + my ($imgWidth, $imgHeight) = $newAsset->getStorageLocation->getSizeInPixels($filename); + my $maxImageSize = $session->setting->get('maxImageSize'); + + if (($imgWidth > $imgHeight) && ($imgWidth > $maxImageSize)) { + $newAsset->getStorageLocation->resize($filename, $maxImageSize); + } + elsif ($imgHeight > $maxImageSize) { + $newAsset->getStorageLocation->resize($filename, , $maxImageSize); + } + } + $newAsset->setSize($storage->getFileSize($filename)); + + setPrivilege($storage->getPath()); } print "Finished adding.\n" unless ($quiet); } #----------------------------------------- -# setPrivileges() +# setPrivilege(path) #----------------------------------------- -sub setPrivileges { - print "Setting filesystem privileges.\n" unless ($quiet); +sub setPrivilege { + my $path = shift; + print "\t\tSetting filesystem privilege. " unless ($quiet); if ($session->os->get("type") eq "Linuxish") { - unless (system("chown -R ".$webUser." ".$session->config->get("uploadsPath"))) { + unless (system("chown -R ".$webUser." ". $path)) { print "Privileges set.\n" unless ($quiet); } else { @@ -294,9 +319,8 @@ sub buildFileListWrap { my ($path) = @_; my (@filelist); - print "Building file list.\n" unless ($quiet); - @filelist = &buildFileList($now,$path); - # &filelistCheck(@filelist); + print "Building file list." unless ($quiet); + @filelist = buildFileList($now,$path); print "File list complete.\n" unless ($quiet); return \@filelist; @@ -316,32 +340,34 @@ sub buildFileList { next if ($file eq "." || $file eq ".."); my $fullpathfile = "$path$slash$file"; if (-f "$fullpathfile") { - $file =~ /(.*?)\.(.*?)$/; + $file =~ /(.*?)\.([^.]*?)$/; $filename = $1; $ext = $2; # TB : filter process : skip files due to options : skipOlderThan and findByExt - next if (&skipFilter($fullpathfile,$ext,$now)); - - # TB : due to recursive search. Not really nice. - my $relpath = &trailFile("$path"); + next if (skipFilter($fullpathfile,$ext,$now)); # TB : check is the filelist doesn't contains two times the same file (file with the same name) # due to recursive call, this can happen. if (exists $filelisthash{$file}) { + print "Error: file \"$file\" exists at several locations. " + . "Both \"$filelisthash{$file}\" and \"$fullpathfile\" contain it. " + . "Exit at the first error of this type.\n" unless ($quiet); - print "Error: file \"$file\" exists at several locations. - Both \"$filelisthash{$file}\" and \"$relpath\" contains it. - Exit at the first error of this type.\n" unless ($quiet); exit 2; } - push(@filelist,{ext=>$ext, filename=>$file, relpath=>$relpath}); - $filelisthash{$file} = $relpath; + push(@filelist, { + ext=>$ext, + filename=>$file, + fullPathFile => $fullpathfile, + }); + + $filelisthash{$file} = $fullpathfile; print "Found file $file.\n" unless ($quiet); } # TB : the recursive call - push(@filelist,&buildFileList($now,"$fullpathfile")) if ((-d "$fullpathfile") && $recursive); + push(@filelist, buildFileList($now,"$fullpathfile")) if ((-d "$fullpathfile") && $recursive); } closedir(FILES); @@ -353,18 +379,6 @@ sub buildFileList { } } -#----------------------------------------- -# getType(filename) -#----------------------------------------- -sub getType { - my ($extension); - $extension = $_[0]; - $extension =~ s/.*\.(.*?)$/$1/; - - return $extension; -} - - #----------------------------------------- # skipFilter(file,ext,time) #----------------------------------------- @@ -391,62 +405,3 @@ sub skipFilter { return 0; } - -#----------------------------------------- -# trailFile(path) -#----------------------------------------- -sub trailFile { - # TB : this is ugly, but unable to find a better way to do it. - # this is supposed to find the relative path to pathToFiles - my ($path) = @_; - my $pathToFiles_temp = $pathToFiles; - - if ($^O =~ /Win/) { - $pathToFiles_temp =~ s/\\/\\\\/g; - } - else { - $pathToFiles_temp =~ s/\//\\\//g; - } - - $path =~ s/$pathToFiles_temp//; - $path =~ s/.//; - $path = "." if ($path eq ""); - - return $path; -} - - -#----------------------------------------- -# checkAssetExists(self,url) -#----------------------------------------- -sub checkAssetExists { - my ($url) = @_; - my $replaceAsset = 0; - my $replaceAssetId = ""; - my $child; - - if (exists $ListAssetExists{$url}) { - $child = $ListAssetExists{$url}; - $replaceAsset = 1; - $replaceAssetId = $child->getId; - } - - return($replaceAsset, $replaceAssetId, $child); -} - - -#----------------------------------------- -# buildListAssetExists(self,url) -#----------------------------------------- -sub buildListAssetExists { - my ($parent) = @_; - my @allelement = $parent->getLineage(); - - foreach my $elem ( @{$allelement[0]} ) { - my $child = WebGUI::Asset::File->newByDynamicClass($session,$elem); - if (defined $child) { - $ListAssetExists{$child->getUrl} = $child; - } - } -} -