From a3d8c390bd165fe6bf3a1ed4db8a26b58be6c1ad Mon Sep 17 00:00:00 2001 From: Colin Kuskie Date: Thu, 23 Jul 2009 21:51:45 +0000 Subject: [PATCH] Fix CDN working with GUID style, vs hex style, storage locations. Add tests for geting a GUID storage location. --- docs/changelog/7.x.x.txt | 1 + lib/WebGUI/Storage.pm | 39 ++++++++++++++++++++++++++++++--------- sbin/syncToCdn.pl | 9 +++++++-- t/Storage.t | 22 +++++++++++++++++++--- 4 files changed, 57 insertions(+), 14 deletions(-) diff --git a/docs/changelog/7.x.x.txt b/docs/changelog/7.x.x.txt index f34f7fd68..dce456241 100644 --- a/docs/changelog/7.x.x.txt +++ b/docs/changelog/7.x.x.txt @@ -30,6 +30,7 @@ - fixed #10695: Adding a new article creates a new version tag - fixed #10693: double titels in help for Story Archive view template - fixed Story Archive sub-folders should not use the failsafe style. + - fixed #10617: synchToCdn.pl misses many files (Wes Morgan, U.S. PIRG) 7.7.15 - fixed #10629: WebGUI::ProfileField create new field bug diff --git a/lib/WebGUI/Storage.pm b/lib/WebGUI/Storage.pm index 99162e1a5..6bb1fc50a 100644 --- a/lib/WebGUI/Storage.pm +++ b/lib/WebGUI/Storage.pm @@ -121,7 +121,7 @@ sub _cdnAdd { my $cdnCfg = $self->session->config->get('cdn'); if ( $cdnCfg and $cdnCfg->{'enabled'} ) { if ( $cdnCfg->{'queuePath'} ) { - my $cdnFile = $cdnCfg->{'queuePath'} . '/' . $self->getHexId; + my $cdnFile = $cdnCfg->{'queuePath'} . '/' . $self->getDirectoryId; my $dest; if ( open $dest, '>', $cdnFile ) { close $dest; # created empty file @@ -161,7 +161,7 @@ sub _cdnDel { unlink $cdnFile; } if ( $cdnCfg->{'queuePath'} ) { - $cdnFile = $cdnCfg->{'queuePath'} . '/' . $self->getHexId; + $cdnFile = $cdnCfg->{'queuePath'} . '/' . $self->getDirectoryId; my $dest; if ( open $dest, '>', $cdnFile ) { print $dest "deleted\n"; @@ -718,7 +718,7 @@ sub deleteFromCdn { and $cdnCfg->{'enabled'} and $cdnCfg->{'syncProgram'} ) { - my $id = $self->getHexId; + my $id = $self->getDirectoryId; my $cmd = sprintf( $cdnCfg->{'deleteProgram'}, $id ); if ( $cmd =~ /$id/ ) { # sanity check, no rm -rf / system($cmd); @@ -726,7 +726,7 @@ sub deleteFromCdn { $self->_addError("Error running CDN deleteProgram: $?"); } if ( $cdnCfg->{'queuePath'} ) { - unlink $cdnCfg->{'queuePath'} . '/' . $self->getHexId; + unlink $cdnCfg->{'queuePath'} . '/' . $id; } } else { # Presume configuration error, missing %s @@ -855,7 +855,14 @@ sub getCdnFileIterator { my $sub = sub { my $id = shift @ids; return if !$id; - return $class->get( $session, $session->id->fromHex($id) ); + my $storageId; + if (length($id) > 22) { + # convert from hex + $storageId = $session->id->fromHex($id); + } else { + $storageId = $id; + } + return $class->get( $session, $storageId ); }; return $sub; } @@ -1273,12 +1280,12 @@ sub getUrl { { if ( $cdnCfg->{'sslUrl'} ) { substr( $cdnCfg->{'sslUrl'}, -1 ) eq '/' and $sep = ''; - $url = $cdnCfg->{'sslUrl'} . $sep . $self->getHexId; + $url = $cdnCfg->{'sslUrl'} . $sep . $self->getDirectoryId; } # else do NOT override $url with CDN URL ($url = $sslUrl || $url) } else { substr( $cdnCfg->{'url'}, -1 ) eq '/' and $sep = ''; - $url = $cdnCfg->{'url'} . $sep . $self->getHexId; + $url = $cdnCfg->{'url'} . $sep . $self->getDirectoryId; } } ## end if ( $cdnCfg and $cdnCfg... if ( defined $file ) { @@ -1696,13 +1703,13 @@ sub syncToCdn { my $originalDir = Cwd::cwd(); my $locDir = join '/', $self->session->config->get('uploadsPath'), @{ $self->{_pathParts} }[ 0 .. 1 ]; chdir $locDir or croak 'Unable to chdir to ' . $locDir . " : $!"; - my $cmd = sprintf( $cdnCfg->{'syncProgram'}, $self->getHexId ); + my $cmd = sprintf( $cdnCfg->{'syncProgram'}, $self->getDirectoryId ); system($cmd); if ($?) { $self->_addError("Error running CDN syncProgram: $?"); } elsif ( $cdnCfg->{'queuePath'} ) { - unlink $cdnCfg->{'queuePath'} . '/' . $self->getHexId; + unlink $cdnCfg->{'queuePath'} . '/' . $self->getDirectoryId; } chdir $originalDir; my $dest; @@ -1784,5 +1791,19 @@ sub untar { return $temp; } +#------------------------------------------------------------------- + +=head2 getDirectoryId ( ) + +Returns the id in base64 or hex depending on how it's stored in +the uploads path. + +=cut + +sub getDirectoryId { + my $self = shift; + return $self->{_pathParts}[2]; +} + 1; diff --git a/sbin/syncToCdn.pl b/sbin/syncToCdn.pl index 980246dba..067396251 100644 --- a/sbin/syncToCdn.pl +++ b/sbin/syncToCdn.pl @@ -138,7 +138,7 @@ sub syncQueue { my $cdnCfg = shift; my $locIter = WebGUI::Storage->getCdnFileIterator($session); while ( my $store = $locIter->() ) { - my $ctrlFile = $cdnCfg->{'queuePath'} . '/' . $session->id->toHex( $store->getId ); + my $ctrlFile = $cdnCfg->{'queuePath'} . '/' . $store->getDirectoryId; if ( -r $ctrlFile and -s $ctrlFile < 12 ) { if ( !-s $ctrlFile ) { # Empty means sync/add/update $store->syncToCdn; @@ -183,7 +183,12 @@ sub syncUploads { if ( opendir my $S2, "$uDir/$subdir/$sub2" ) { my @fileId = grep { !/^\.+$/ } readdir($S2); foreach my $fileId (@fileId) { - my $store = WebGUI::Storage->get( $session, $session->id->fromHex($fileId) ); + my $storageId = $fileId; + if (length($storageId) > 22) { + # need to convert from hex + $storageId = $session->id->fromHex($storageId); + } + my $store = WebGUI::Storage->get( $session, $storageId ); $store->syncToCdn; # here is the meat } close $S2; diff --git a/t/Storage.t b/t/Storage.t index 5ebf5def7..55d33911e 100644 --- a/t/Storage.t +++ b/t/Storage.t @@ -23,6 +23,7 @@ use Test::Deep; use Test::MockObject; use Cwd; use Data::Dumper; +use Path::Class::Dir; my $session = WebGUI::Test->session; @@ -30,7 +31,7 @@ my $cwd = Cwd::cwd(); my ($extensionTests, $fileIconTests) = setupDataDrivenTests($session); -my $numTests = 122; # increment this value for each test you create +my $numTests = 136; # increment this value for each test you create plan tests => $numTests + scalar @{ $extensionTests } + scalar @{ $fileIconTests }; my $uploadDir = $session->config->get('uploadsPath'); @@ -62,11 +63,26 @@ is( $storage1->getLastError, undef, "No errors during path creation"); #################################################### # -# getPathFrag +# getPathFrag, getDirectoryId, get # #################################################### -is( $storage1->getPathFrag, '7e/8a/7e8a1b6a', 'pathFrag returns correct value'); +is( $storage1->getPathFrag, '7e/8a/7e8a1b6a', 'pathFrag returns correct value'); +is( $storage1->getDirectoryId, '7e8a1b6a', 'getDirectoryId returns the last path element'); + +##Build an old-style GUID storage location +my $uploadsBase = Path::Class::Dir->new($uploadDir); +my $newGuid = $session->id->generate(); +my @guidPathParts = (substr($newGuid, 0, 2), substr($newGuid, 2, 2), $newGuid); +my $guidDir = $uploadsBase->subdir(@guidPathParts); +$guidDir->mkpath(0); +ok(-e $guidDir->stringify, 'created GUID storage location for backwards compatibility testing'); + +my $guidStorage = WebGUI::Storage->get($session, $newGuid); +WebGUI::Test->storagesToDelete($guidStorage); +isa_ok($guidStorage, 'WebGUI::Storage'); +is($guidStorage->getId, $newGuid, 'GUID storage has correct id'); +is($guidStorage->getDirectoryId, $newGuid, '... getDirectoryId'); #################################################### #