oqapi/webgui
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fileImport fixes.

Browse source
This commit is contained in:
Martin Kamerbeek 2007-02-26 21:43:53 +00:00
parent 5aa21faaa3
commit 21573a54f1
2 changed files with 110 additions and 153 deletions
Download patch file Download diff file Expand all files Collapse all files

4
docs/changelog/7.x.x.txt
View file

@ -6,7 +6,9 @@
- fix: Off by 1 error while displaying ad impressions (perlDreamer Consulting, LLC) - fix: Off by 1 error while displaying ad impressions (perlDreamer Consulting, LLC)
- fix: Wrong link in template toolkit help (perlDreamer Consulting, LLC) - fix: Wrong link in template toolkit help (perlDreamer Consulting, LLC)
- fix: Outdated info in Env macro help (perlDreamer Consulting, LLC) - fix: Outdated info in Env macro help (perlDreamer Consulting, LLC)
- fix: A bunch of bugs in fileImport.pl and improved its performance very
significantly (Martin Kamerbeek / Oqapi)
7.3.10 7.3.10
- fix: Calendar Update Feeds now handles iCalendar "DURATION" field - fix: Calendar Update Feeds now handles iCalendar "DURATION" field
- fix: Various potential bugs in WebGUI::DateTime due to inheritence from - fix: Various potential bugs in WebGUI::DateTime due to inheritence from

259
sbin/fileImport.pl
View file

@ -53,31 +53,31 @@ my $overwrite = '';
my $ignoreExtInName = ''; my $ignoreExtInName = '';
GetOptions( GetOptions(
'configFile=s'=>\$configFile, 'configFile=s' => \$configFile,
'owner=s'=>\$owner, 'owner=s' => \$owner,
'groupToView=s'=>\$groupToView, 'groupToView=s' => \$groupToView,
'groupToEdit=s'=>\$groupToEdit, 'groupToEdit=s' => \$groupToEdit,
'help'=>\$help, 'help' => \$help,
'override'=>\$override, 'override' => \$override,
'pathToFiles=s'=>\$pathToFiles, 'pathToFiles=s' => \$pathToFiles,
'quiet'=>\$quiet, 'quiet' => \$quiet,
'webUser=s'=>\$webUser, 'webUser=s' => \$webUser,
'parentAssetId=s'=>\$parentAssetId, 'parentAssetId=s' => \$parentAssetId,
'skipOlderThan=i'=>\$skipOlderThan, 'skipOlderThan=i' => \$skipOlderThan,
'findByExt=s'=>\$findByExt, 'findByExt=s' => \$findByExt,
'recursive'=>\$recursive, 'recursive' => \$recursive,
'overwrite'=>\$overwrite, 'overwrite' => \$overwrite,
'ignoreExtInName'=>\$ignoreExtInName 'ignoreExtInName' => \$ignoreExtInName
); );
if ($configFile eq "" || $pathToFiles eq "" || $parentAssetId eq "") { if ($configFile eq "" || $pathToFiles eq "" || $parentAssetId eq "") {
&printHelp; printHelp();
exit 4; exit 4;
}; };
if ($help) { if ($help) {
&printHelp; printHelp();
exit 1; exit 1;
} }
@ -185,17 +185,19 @@ $session->user({userId=>3});
print "OK\n" unless ($quiet); print "OK\n" unless ($quiet);
my $parent = WebGUI::Asset::File->newByDynamicClass($session, $parentAssetId); my $parent = WebGUI::Asset::File->newByDynamicClass($session, $parentAssetId);
if (defined $parent) { unless (defined $parent) {
&buildListAssetExists($parent);
} else {
print "Warning: Parent asset '".$parentAssetId."' does not exist. Cannot import files.\n"; print "Warning: Parent asset '".$parentAssetId."' does not exist. Cannot import files.\n";
exit 5; exit 5;
} }
print "End of the childs detection\n" unless ($quiet); print "End of the childs detection\n" unless ($quiet);
# TB : wrap buildFileList in buildFileListWrap function for recursive search. # TB : wrap buildFileList in buildFileListWrap function for recursive search.
&addFiles(&buildFileListWrap($pathToFiles)); addFiles(buildFileListWrap($pathToFiles));
&setPrivileges();
print "Committing version tag..." unless ($quiet);
my $versionTag = WebGUI::VersionTag->getWorking($session);
$versionTag->commit;
print "Done.\n" unless ($quiet);
print "Cleaning up..." unless ($quiet); print "Cleaning up..." unless ($quiet);
$session->var->end(); $session->var->end();
@ -208,74 +210,97 @@ exit 0;
# addFiles(dbHandler, filelistHashRef, webguiSettingsHashRef, pathToCopyFrom) # addFiles(dbHandler, filelistHashRef, webguiSettingsHashRef, pathToCopyFrom)
#----------------------------------------- #-----------------------------------------
sub addFiles { sub addFiles {
my ($class, $templateId, $url, $newAsset);
my $filelist = shift; my $filelist = shift;
print "Adding files...\n" unless ($quiet); print "Adding files...\n" unless ($quiet);
foreach my $file (@{$filelist}) { foreach my $file (@{$filelist}) {
my $class = 'WebGUI::Asset::File'; print "\tAdding ".$file->{fullPathFile}." to the database.\n" unless ($quiet);
# TB : add the path relative to $pathToFile in the message. # Figure out whether the file is an image or not by its extension.
print "\tAdding ".$file->{relpath}.$slash.$file->{filename}." to the database.\n" unless ($quiet);
my $templateId = 'PBtmpl0000000000000024';
if (isIn(lc($file->{ext}),@nailable)) { if (isIn(lc($file->{ext}),@nailable)) {
$class = 'WebGUI::Asset::File::Image'; $class = 'WebGUI::Asset::File::Image';
$templateId = 'PBtmpl0000000000000088' $templateId = 'PBtmpl0000000000000088';
}
else {
$class = 'WebGUI::Asset::File';
$templateId = 'PBtmpl0000000000000024';
} }
my $url = $parent->getUrl.'/'.$file->{filename};
# TB : Possibly detect if the Asset exists already. $url = $parent->getUrl.'/'.$file->{filename};
my ($replaceAsset, $replaceAssetId, $child) = (0,"",""); $url =~ s{^/}{};
($replaceAsset, $replaceAssetId, $child) = &checkAssetExists($url) if ($overwrite);
if ($replaceAsset == 1) { # Create a new storage location and add the file to it.
# TB : If the Asset exists, just copy the file. my $storage = WebGUI::Storage->create($session);
# To be check. my $filename = $storage->addFileFromFilesystem($file->{fullPathFile});
my $storage = WebGUI::Storage->get($session, $replaceAssetId);
# TB : possibly remove the extension if the ignoreExtInName feature enabled.
my $filenameTitle = $filename;
$filenameTitle =~ s/\.$file->{ext}// if ($ignoreExtInName);
# Set up the properties of the file to be added.
my $assetProperties = {
className => $class,
title => $filenameTitle,
menuTitle => $filenameTitle,
filename => $filename,
storageId => $storage->getId,
isHidden => 1,
url => $url,
groupIdView => $groupToView,
groupIdEdit => $groupToEdit,
templateId => $templateId,
endDate => 32472169200,
ownerUserId => $owner,
};
if (WebGUI::Asset->urlExists($session, $url) && $overwrite) {
print "\t\tAsset exists already. Replace the file.\n" unless ($quiet); print "\t\tAsset exists already. Replace the file.\n" unless ($quiet);
my $filename = $storage->addFileFromFilesystem("$pathToFiles$slash$file->{relpath}$slash$file->{filename}");
$child->generateThumbnail if ($class eq 'WebGUI::Asset::File::Image'); # Add a new revision.
$child->setSize($storage->getFileSize($filename)); my $originalAsset = WebGUI::Asset->newByUrl($session, $url);
$newAsset = $originalAsset->addRevision($assetProperties);
} }
else { else {
print "\t\tCreate the new asset.\n" unless ($quiet); print "\t\tCreate the new asset.\n" unless ($quiet);
my $storage = WebGUI::Storage->create($session);
my $filename = $storage->addFileFromFilesystem("$pathToFiles$slash$file->{relpath}$slash$file->{filename}");
# TB : possibly remove the extension if the ignoreExtInName feature enabled.
my $filenameTitle = $filename;
$filenameTitle =~ s/\.$file->{ext}// if ($ignoreExtInName);
my $newAsset = $parent->addChild({ # Add a new asset.
className => $class, $newAsset = $parent->addChild($assetProperties);
title => $filenameTitle,
menuTitle => $filenameTitle,
filename => $filename,
storageId => $storage->getId,
isHidden => 1,
url => $url,
groupIdView => $groupToView,
groupIdEdit => $groupToEdit,
templateId => $templateId,
endDate => 32472169200,
ownerUserId => $owner,
});
$newAsset->generateThumbnail if ($class eq 'WebGUI::Asset::File::Image');
$newAsset->setSize($storage->getFileSize($filename));
} }
# Create thumbnail, scale and set size.
if ($class eq 'WebGUI::Asset::File::Image') {
# Generate thumbnail
$newAsset->generateThumbnail($session->setting->get('thumbnailSize'));
# Resize image to maxImageSize if necessary.
my ($imgWidth, $imgHeight) = $newAsset->getStorageLocation->getSizeInPixels($filename);
my $maxImageSize = $session->setting->get('maxImageSize');
if (($imgWidth > $imgHeight) && ($imgWidth > $maxImageSize)) {
$newAsset->getStorageLocation->resize($filename, $maxImageSize);
}
elsif ($imgHeight > $maxImageSize) {
$newAsset->getStorageLocation->resize($filename, , $maxImageSize);
}
}
$newAsset->setSize($storage->getFileSize($filename));
setPrivilege($storage->getPath());
} }
print "Finished adding.\n" unless ($quiet); print "Finished adding.\n" unless ($quiet);
} }
#----------------------------------------- #-----------------------------------------
# setPrivileges() # setPrivilege(path)
#----------------------------------------- #-----------------------------------------
sub setPrivileges { sub setPrivilege {
print "Setting filesystem privileges.\n" unless ($quiet); my $path = shift;
print "\t\tSetting filesystem privilege. " unless ($quiet);
if ($session->os->get("type") eq "Linuxish") { if ($session->os->get("type") eq "Linuxish") {
unless (system("chown -R ".$webUser." ".$session->config->get("uploadsPath"))) { unless (system("chown -R ".$webUser." ". $path)) {
print "Privileges set.\n" unless ($quiet); print "Privileges set.\n" unless ($quiet);
} }
else { else {
@ -294,9 +319,8 @@ sub buildFileListWrap {
my ($path) = @_; my ($path) = @_;
my (@filelist); my (@filelist);
print "Building file list.\n" unless ($quiet); print "Building file list." unless ($quiet);
@filelist = &buildFileList($now,$path); @filelist = buildFileList($now,$path);
# &filelistCheck(@filelist);
print "File list complete.\n" unless ($quiet); print "File list complete.\n" unless ($quiet);
return \@filelist; return \@filelist;
@ -316,32 +340,34 @@ sub buildFileList {
next if ($file eq "." || $file eq ".."); next if ($file eq "." || $file eq "..");
my $fullpathfile = "$path$slash$file"; my $fullpathfile = "$path$slash$file";
if (-f "$fullpathfile") { if (-f "$fullpathfile") {
$file =~ /(.*?)\.(.*?)$/; $file =~ /(.*?)\.([^.]*?)$/;
$filename = $1; $filename = $1;
$ext = $2; $ext = $2;
# TB : filter process : skip files due to options : skipOlderThan and findByExt # TB : filter process : skip files due to options : skipOlderThan and findByExt
next if (&skipFilter($fullpathfile,$ext,$now)); next if (skipFilter($fullpathfile,$ext,$now));
# TB : due to recursive search. Not really nice.
my $relpath = &trailFile("$path");
# TB : check is the filelist doesn't contains two times the same file (file with the same name) # TB : check is the filelist doesn't contains two times the same file (file with the same name)
# due to recursive call, this can happen. # due to recursive call, this can happen.
if (exists $filelisthash{$file}) { if (exists $filelisthash{$file}) {
print "Error: file \"$file\" exists at several locations. "
. "Both \"$filelisthash{$file}\" and \"$fullpathfile\" contain it. "
. "Exit at the first error of this type.\n" unless ($quiet);
print "Error: file \"$file\" exists at several locations.
Both \"$filelisthash{$file}\" and \"$relpath\" contains it.
Exit at the first error of this type.\n" unless ($quiet);
exit 2; exit 2;
} }
push(@filelist,{ext=>$ext, filename=>$file, relpath=>$relpath}); push(@filelist, {
$filelisthash{$file} = $relpath; ext=>$ext,
filename=>$file,
fullPathFile => $fullpathfile,
});
$filelisthash{$file} = $fullpathfile;
print "Found file $file.\n" unless ($quiet); print "Found file $file.\n" unless ($quiet);
} }
# TB : the recursive call # TB : the recursive call
push(@filelist,&buildFileList($now,"$fullpathfile")) if ((-d "$fullpathfile") && $recursive); push(@filelist, buildFileList($now,"$fullpathfile")) if ((-d "$fullpathfile") && $recursive);
} }
closedir(FILES); closedir(FILES);
@ -353,18 +379,6 @@ sub buildFileList {
} }
} }
#-----------------------------------------
# getType(filename)
#-----------------------------------------
sub getType {
my ($extension);
$extension = $_[0];
$extension =~ s/.*\.(.*?)$/$1/;
return $extension;
}
#----------------------------------------- #-----------------------------------------
# skipFilter(file,ext,time) # skipFilter(file,ext,time)
#----------------------------------------- #-----------------------------------------
@ -391,62 +405,3 @@ sub skipFilter {
return 0; return 0;
} }
#-----------------------------------------
# trailFile(path)
#-----------------------------------------
sub trailFile {
# TB : this is ugly, but unable to find a better way to do it.
# this is supposed to find the relative path to pathToFiles
my ($path) = @_;
my $pathToFiles_temp = $pathToFiles;
if ($^O =~ /Win/) {
$pathToFiles_temp =~ s/\\/\\\\/g;
}
else {
$pathToFiles_temp =~ s/\//\\\//g;
}
$path =~ s/$pathToFiles_temp//;
$path =~ s/.//;
$path = "." if ($path eq "");
return $path;
}
#-----------------------------------------
# checkAssetExists(self,url)
#-----------------------------------------
sub checkAssetExists {
my ($url) = @_;
my $replaceAsset = 0;
my $replaceAssetId = "";
my $child;
if (exists $ListAssetExists{$url}) {
$child = $ListAssetExists{$url};
$replaceAsset = 1;
$replaceAssetId = $child->getId;
}
return($replaceAsset, $replaceAssetId, $child);
}
#-----------------------------------------
# buildListAssetExists(self,url)
#-----------------------------------------
sub buildListAssetExists {
my ($parent) = @_;
my @allelement = $parent->getLineage();
foreach my $elem ( @{$allelement[0]} ) {
my $child = WebGUI::Asset::File->newByDynamicClass($session,$elem);
if (defined $child) {
$ListAssetExists{$child->getUrl} = $child;
}
}
}