fileImport fixes.
This commit is contained in:
Martin Kamerbeek
parent
5aa21faaa3
commit
21573a54f1
2 changed files with 110 additions and 153 deletions
|
|
@ -6,7 +6,9 @@
|
|||
- fix: Off by 1 error while displaying ad impressions (perlDreamer Consulting, LLC)
|
||||
- fix: Wrong link in template toolkit help (perlDreamer Consulting, LLC)
|
||||
- fix: Outdated info in Env macro help (perlDreamer Consulting, LLC)
|
||||
|
||||
- fix: A bunch of bugs in fileImport.pl and improved its performance very
|
||||
significantly (Martin Kamerbeek / Oqapi)
|
||||
|
||||
7.3.10
|
||||
- fix: Calendar Update Feeds now handles iCalendar "DURATION" field
|
||||
- fix: Various potential bugs in WebGUI::DateTime due to inheritence from
|
||||
|
|
|
|||
|
|
@ -53,31 +53,31 @@ my $overwrite = '';
|
|||
my $ignoreExtInName = '';
|
||||
|
||||
GetOptions(
|
||||
'configFile=s'=>\$configFile,
|
||||
'owner=s'=>\$owner,
|
||||
'groupToView=s'=>\$groupToView,
|
||||
'groupToEdit=s'=>\$groupToEdit,
|
||||
'help'=>\$help,
|
||||
'override'=>\$override,
|
||||
'pathToFiles=s'=>\$pathToFiles,
|
||||
'quiet'=>\$quiet,
|
||||
'webUser=s'=>\$webUser,
|
||||
'parentAssetId=s'=>\$parentAssetId,
|
||||
'skipOlderThan=i'=>\$skipOlderThan,
|
||||
'findByExt=s'=>\$findByExt,
|
||||
'recursive'=>\$recursive,
|
||||
'overwrite'=>\$overwrite,
|
||||
'ignoreExtInName'=>\$ignoreExtInName
|
||||
'configFile=s' => \$configFile,
|
||||
'owner=s' => \$owner,
|
||||
'groupToView=s' => \$groupToView,
|
||||
'groupToEdit=s' => \$groupToEdit,
|
||||
'help' => \$help,
|
||||
'override' => \$override,
|
||||
'pathToFiles=s' => \$pathToFiles,
|
||||
'quiet' => \$quiet,
|
||||
'webUser=s' => \$webUser,
|
||||
'parentAssetId=s' => \$parentAssetId,
|
||||
'skipOlderThan=i' => \$skipOlderThan,
|
||||
'findByExt=s' => \$findByExt,
|
||||
'recursive' => \$recursive,
|
||||
'overwrite' => \$overwrite,
|
||||
'ignoreExtInName' => \$ignoreExtInName
|
||||
);
|
||||
|
||||
|
||||
if ($configFile eq "" || $pathToFiles eq "" || $parentAssetId eq "") {
|
||||
&printHelp;
|
||||
printHelp();
|
||||
exit 4;
|
||||
};
|
||||
|
||||
if ($help) {
|
||||
&printHelp;
|
||||
printHelp();
|
||||
exit 1;
|
||||
}
|
||||
|
||||
|
|
@ -185,17 +185,19 @@ $session->user({userId=>3});
|
|||
print "OK\n" unless ($quiet);
|
||||
|
||||
my $parent = WebGUI::Asset::File->newByDynamicClass($session, $parentAssetId);
|
||||
if (defined $parent) {
|
||||
&buildListAssetExists($parent);
|
||||
} else {
|
||||
unless (defined $parent) {
|
||||
print "Warning: Parent asset '".$parentAssetId."' does not exist. Cannot import files.\n";
|
||||
exit 5;
|
||||
}
|
||||
print "End of the childs detection\n" unless ($quiet);
|
||||
|
||||
# TB : wrap buildFileList in buildFileListWrap function for recursive search.
|
||||
&addFiles(&buildFileListWrap($pathToFiles));
|
||||
&setPrivileges();
|
||||
addFiles(buildFileListWrap($pathToFiles));
|
||||
|
||||
print "Committing version tag..." unless ($quiet);
|
||||
my $versionTag = WebGUI::VersionTag->getWorking($session);
|
||||
$versionTag->commit;
|
||||
print "Done.\n" unless ($quiet);
|
||||
|
||||
print "Cleaning up..." unless ($quiet);
|
||||
$session->var->end();
|
||||
|
|
@ -208,74 +210,97 @@ exit 0;
|
|||
# addFiles(dbHandler, filelistHashRef, webguiSettingsHashRef, pathToCopyFrom)
|
||||
#-----------------------------------------
|
||||
sub addFiles {
|
||||
my ($class, $templateId, $url, $newAsset);
|
||||
my $filelist = shift;
|
||||
|
||||
print "Adding files...\n" unless ($quiet);
|
||||
foreach my $file (@{$filelist}) {
|
||||
my $class = 'WebGUI::Asset::File';
|
||||
|
||||
# TB : add the path relative to $pathToFile in the message.
|
||||
print "\tAdding ".$file->{relpath}.$slash.$file->{filename}." to the database.\n" unless ($quiet);
|
||||
|
||||
my $templateId = 'PBtmpl0000000000000024';
|
||||
print "\tAdding ".$file->{fullPathFile}." to the database.\n" unless ($quiet);
|
||||
|
||||
# Figure out whether the file is an image or not by its extension.
|
||||
if (isIn(lc($file->{ext}),@nailable)) {
|
||||
$class = 'WebGUI::Asset::File::Image';
|
||||
$templateId = 'PBtmpl0000000000000088'
|
||||
$templateId = 'PBtmpl0000000000000088';
|
||||
}
|
||||
else {
|
||||
$class = 'WebGUI::Asset::File';
|
||||
$templateId = 'PBtmpl0000000000000024';
|
||||
}
|
||||
my $url = $parent->getUrl.'/'.$file->{filename};
|
||||
|
||||
# TB : Possibly detect if the Asset exists already.
|
||||
my ($replaceAsset, $replaceAssetId, $child) = (0,"","");
|
||||
($replaceAsset, $replaceAssetId, $child) = &checkAssetExists($url) if ($overwrite);
|
||||
$url = $parent->getUrl.'/'.$file->{filename};
|
||||
$url =~ s{^/}{};
|
||||
|
||||
if ($replaceAsset == 1) {
|
||||
# TB : If the Asset exists, just copy the file.
|
||||
# To be check.
|
||||
my $storage = WebGUI::Storage->get($session, $replaceAssetId);
|
||||
# Create a new storage location and add the file to it.
|
||||
my $storage = WebGUI::Storage->create($session);
|
||||
my $filename = $storage->addFileFromFilesystem($file->{fullPathFile});
|
||||
|
||||
# TB : possibly remove the extension if the ignoreExtInName feature enabled.
|
||||
my $filenameTitle = $filename;
|
||||
$filenameTitle =~ s/\.$file->{ext}// if ($ignoreExtInName);
|
||||
|
||||
# Set up the properties of the file to be added.
|
||||
my $assetProperties = {
|
||||
className => $class,
|
||||
title => $filenameTitle,
|
||||
menuTitle => $filenameTitle,
|
||||
filename => $filename,
|
||||
storageId => $storage->getId,
|
||||
isHidden => 1,
|
||||
url => $url,
|
||||
groupIdView => $groupToView,
|
||||
groupIdEdit => $groupToEdit,
|
||||
templateId => $templateId,
|
||||
endDate => 32472169200,
|
||||
ownerUserId => $owner,
|
||||
};
|
||||
|
||||
if (WebGUI::Asset->urlExists($session, $url) && $overwrite) {
|
||||
print "\t\tAsset exists already. Replace the file.\n" unless ($quiet);
|
||||
my $filename = $storage->addFileFromFilesystem("$pathToFiles$slash$file->{relpath}$slash$file->{filename}");
|
||||
$child->generateThumbnail if ($class eq 'WebGUI::Asset::File::Image');
|
||||
$child->setSize($storage->getFileSize($filename));
|
||||
|
||||
# Add a new revision.
|
||||
my $originalAsset = WebGUI::Asset->newByUrl($session, $url);
|
||||
$newAsset = $originalAsset->addRevision($assetProperties);
|
||||
}
|
||||
else {
|
||||
print "\t\tCreate the new asset.\n" unless ($quiet);
|
||||
my $storage = WebGUI::Storage->create($session);
|
||||
my $filename = $storage->addFileFromFilesystem("$pathToFiles$slash$file->{relpath}$slash$file->{filename}");
|
||||
|
||||
# TB : possibly remove the extension if the ignoreExtInName feature enabled.
|
||||
my $filenameTitle = $filename;
|
||||
$filenameTitle =~ s/\.$file->{ext}// if ($ignoreExtInName);
|
||||
|
||||
my $newAsset = $parent->addChild({
|
||||
className => $class,
|
||||
title => $filenameTitle,
|
||||
menuTitle => $filenameTitle,
|
||||
filename => $filename,
|
||||
storageId => $storage->getId,
|
||||
isHidden => 1,
|
||||
url => $url,
|
||||
groupIdView => $groupToView,
|
||||
groupIdEdit => $groupToEdit,
|
||||
templateId => $templateId,
|
||||
endDate => 32472169200,
|
||||
ownerUserId => $owner,
|
||||
});
|
||||
|
||||
$newAsset->generateThumbnail if ($class eq 'WebGUI::Asset::File::Image');
|
||||
$newAsset->setSize($storage->getFileSize($filename));
|
||||
# Add a new asset.
|
||||
$newAsset = $parent->addChild($assetProperties);
|
||||
}
|
||||
|
||||
# Create thumbnail, scale and set size.
|
||||
if ($class eq 'WebGUI::Asset::File::Image') {
|
||||
# Generate thumbnail
|
||||
$newAsset->generateThumbnail($session->setting->get('thumbnailSize'));
|
||||
|
||||
# Resize image to maxImageSize if necessary.
|
||||
my ($imgWidth, $imgHeight) = $newAsset->getStorageLocation->getSizeInPixels($filename);
|
||||
my $maxImageSize = $session->setting->get('maxImageSize');
|
||||
|
||||
if (($imgWidth > $imgHeight) && ($imgWidth > $maxImageSize)) {
|
||||
$newAsset->getStorageLocation->resize($filename, $maxImageSize);
|
||||
}
|
||||
elsif ($imgHeight > $maxImageSize) {
|
||||
$newAsset->getStorageLocation->resize($filename, , $maxImageSize);
|
||||
}
|
||||
}
|
||||
$newAsset->setSize($storage->getFileSize($filename));
|
||||
|
||||
setPrivilege($storage->getPath());
|
||||
}
|
||||
|
||||
print "Finished adding.\n" unless ($quiet);
|
||||
}
|
||||
|
||||
#-----------------------------------------
|
||||
# setPrivileges()
|
||||
# setPrivilege(path)
|
||||
#-----------------------------------------
|
||||
sub setPrivileges {
|
||||
print "Setting filesystem privileges.\n" unless ($quiet);
|
||||
sub setPrivilege {
|
||||
my $path = shift;
|
||||
print "\t\tSetting filesystem privilege. " unless ($quiet);
|
||||
|
||||
if ($session->os->get("type") eq "Linuxish") {
|
||||
unless (system("chown -R ".$webUser." ".$session->config->get("uploadsPath"))) {
|
||||
unless (system("chown -R ".$webUser." ". $path)) {
|
||||
print "Privileges set.\n" unless ($quiet);
|
||||
}
|
||||
else {
|
||||
|
|
@ -294,9 +319,8 @@ sub buildFileListWrap {
|
|||
my ($path) = @_;
|
||||
my (@filelist);
|
||||
|
||||
print "Building file list.\n" unless ($quiet);
|
||||
@filelist = &buildFileList($now,$path);
|
||||
# &filelistCheck(@filelist);
|
||||
print "Building file list." unless ($quiet);
|
||||
@filelist = buildFileList($now,$path);
|
||||
print "File list complete.\n" unless ($quiet);
|
||||
|
||||
return \@filelist;
|
||||
|
|
@ -316,32 +340,34 @@ sub buildFileList {
|
|||
next if ($file eq "." || $file eq "..");
|
||||
my $fullpathfile = "$path$slash$file";
|
||||
if (-f "$fullpathfile") {
|
||||
$file =~ /(.*?)\.(.*?)$/;
|
||||
$file =~ /(.*?)\.([^.]*?)$/;
|
||||
$filename = $1;
|
||||
$ext = $2;
|
||||
|
||||
# TB : filter process : skip files due to options : skipOlderThan and findByExt
|
||||
next if (&skipFilter($fullpathfile,$ext,$now));
|
||||
|
||||
# TB : due to recursive search. Not really nice.
|
||||
my $relpath = &trailFile("$path");
|
||||
next if (skipFilter($fullpathfile,$ext,$now));
|
||||
|
||||
# TB : check is the filelist doesn't contains two times the same file (file with the same name)
|
||||
# due to recursive call, this can happen.
|
||||
if (exists $filelisthash{$file}) {
|
||||
print "Error: file \"$file\" exists at several locations. "
|
||||
. "Both \"$filelisthash{$file}\" and \"$fullpathfile\" contain it. "
|
||||
. "Exit at the first error of this type.\n" unless ($quiet);
|
||||
|
||||
print "Error: file \"$file\" exists at several locations.
|
||||
Both \"$filelisthash{$file}\" and \"$relpath\" contains it.
|
||||
Exit at the first error of this type.\n" unless ($quiet);
|
||||
exit 2;
|
||||
}
|
||||
|
||||
push(@filelist,{ext=>$ext, filename=>$file, relpath=>$relpath});
|
||||
$filelisthash{$file} = $relpath;
|
||||
push(@filelist, {
|
||||
ext=>$ext,
|
||||
filename=>$file,
|
||||
fullPathFile => $fullpathfile,
|
||||
});
|
||||
|
||||
$filelisthash{$file} = $fullpathfile;
|
||||
print "Found file $file.\n" unless ($quiet);
|
||||
}
|
||||
# TB : the recursive call
|
||||
push(@filelist,&buildFileList($now,"$fullpathfile")) if ((-d "$fullpathfile") && $recursive);
|
||||
push(@filelist, buildFileList($now,"$fullpathfile")) if ((-d "$fullpathfile") && $recursive);
|
||||
}
|
||||
|
||||
closedir(FILES);
|
||||
|
|
@ -353,18 +379,6 @@ sub buildFileList {
|
|||
}
|
||||
}
|
||||
|
||||
#-----------------------------------------
|
||||
# getType(filename)
|
||||
#-----------------------------------------
|
||||
sub getType {
|
||||
my ($extension);
|
||||
$extension = $_[0];
|
||||
$extension =~ s/.*\.(.*?)$/$1/;
|
||||
|
||||
return $extension;
|
||||
}
|
||||
|
||||
|
||||
#-----------------------------------------
|
||||
# skipFilter(file,ext,time)
|
||||
#-----------------------------------------
|
||||
|
|
@ -391,62 +405,3 @@ sub skipFilter {
|
|||
return 0;
|
||||
}
|
||||
|
||||
|
||||
#-----------------------------------------
|
||||
# trailFile(path)
|
||||
#-----------------------------------------
|
||||
sub trailFile {
|
||||
# TB : this is ugly, but unable to find a better way to do it.
|
||||
# this is supposed to find the relative path to pathToFiles
|
||||
my ($path) = @_;
|
||||
my $pathToFiles_temp = $pathToFiles;
|
||||
|
||||
if ($^O =~ /Win/) {
|
||||
$pathToFiles_temp =~ s/\\/\\\\/g;
|
||||
}
|
||||
else {
|
||||
$pathToFiles_temp =~ s/\//\\\//g;
|
||||
}
|
||||
|
||||
$path =~ s/$pathToFiles_temp//;
|
||||
$path =~ s/.//;
|
||||
$path = "." if ($path eq "");
|
||||
|
||||
return $path;
|
||||
}
|
||||
|
||||
|
||||
#-----------------------------------------
|
||||
# checkAssetExists(self,url)
|
||||
#-----------------------------------------
|
||||
sub checkAssetExists {
|
||||
my ($url) = @_;
|
||||
my $replaceAsset = 0;
|
||||
my $replaceAssetId = "";
|
||||
my $child;
|
||||
|
||||
if (exists $ListAssetExists{$url}) {
|
||||
$child = $ListAssetExists{$url};
|
||||
$replaceAsset = 1;
|
||||
$replaceAssetId = $child->getId;
|
||||
}
|
||||
|
||||
return($replaceAsset, $replaceAssetId, $child);
|
||||
}
|
||||
|
||||
|
||||
#-----------------------------------------
|
||||
# buildListAssetExists(self,url)
|
||||
#-----------------------------------------
|
||||
sub buildListAssetExists {
|
||||
my ($parent) = @_;
|
||||
my @allelement = $parent->getLineage();
|
||||
|
||||
foreach my $elem ( @{$allelement[0]} ) {
|
||||
my $child = WebGUI::Asset::File->newByDynamicClass($session,$elem);
|
||||
if (defined $child) {
|
||||
$ListAssetExists{$child->getUrl} = $child;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue