diff --git a/docs/upgrades/upgrade_7.6.14-7.7.0.pl b/docs/upgrades/upgrade_7.6.14-7.7.0.pl index c915b2fe1..c3902cae3 100644 --- a/docs/upgrades/upgrade_7.6.14-7.7.0.pl +++ b/docs/upgrades/upgrade_7.6.14-7.7.0.pl @@ -22,7 +22,8 @@ use Getopt::Long; use WebGUI::Session; use WebGUI::Storage; use WebGUI::Asset; - +use WebGUI::PassiveAnalytics::Rule; +use WebGUI::Utility; my $toVersion = '7.7.0'; my $quiet; # this line required @@ -37,6 +38,14 @@ addScreenshotTemplatesToMatrix( $session ); surveyDoAfterTimeLimit($session); surveyRemoveResponseTemplate($session); +# Passive Analytics +pa_installLoggingTables($session); +pa_installPassiveAnalyticsRule($session); +pa_installPassiveAnalyticsConfig($session); +pa_installWorkflow($session); +pa_addPassiveAnalyticsSettings($session); +pa_addPassiveAnalyticsStatus($session); + finish($session); # this line required #---------------------------------------------------------------------------- @@ -82,6 +91,162 @@ sub surveyRemoveResponseTemplate { print "DONE!\n" unless $quiet; } +sub pa_installLoggingTables { + my $session = shift; + print "\tInstall logging tables... "; + my $db = $session->db; + $db->write(<write(<write(<write(<write(<write(<crud_createTable($session); + print "DONE!\n"; +} + +#---------------------------------------------------------------------------- +# Add the PassiveAnalytics Settings +sub pa_addPassiveAnalyticsSettings { + my $session = shift; + print "\tInstall Passive Analytics settings... "; + # and here's our code + $session->setting->add('passiveAnalyticsInterval', 300); + $session->setting->add('passiveAnalyticsDeleteDelta', 0); + $session->setting->add('passiveAnalyticsEnabled', 0); + print "DONE!\n"; +} + +#---------------------------------------------------------------------------- +# Add the PassiveAnalytics Rule table +sub pa_addPassiveAnalyticsStatus { + my $session = shift; + my $db = $session->db; + print "\tInstall Passive Analytics status table... "; + # and here's our code + $db->write(<write(<write('insert into passiveAnalyticsStatus (userId) VALUES (3)'); + print "DONE!\n"; +} + +#---------------------------------------------------------------------------- +# Add the Passive Analytics config file entry +# for the adminConsole and the content handler +sub pa_installPassiveAnalyticsConfig { + my $session = shift; + print "\tAdd Passive Analytics entry to the config file... "; + # Admin Bar/Console + my $adminConsole = $session->config->get('adminConsole'); + if (!exists $adminConsole->{'passiveAnalytics'}) { + $adminConsole->{'passiveAnalytics'} = { + "icon" => "passiveAnalytics.png", + "uiLevel" => 1, + "url" => "^PageUrl(\"\",op=passiveAnalytics;func=editRuleflow);", + "title" => "^International(Passive Analytics,PassiveAnalytics);", + "groupSetting" => "3", + }; + $session->config->set('adminConsole', $adminConsole); + } + # Content Handler + my $contentHandlers = $session->config->get('contentHandlers'); + if (!isIn('WebGUI::Content::PassiveAnalytics',@{ $contentHandlers} ) ) { + my $contentIndex = 0; + HANDLER: while ($contentIndex <= $#{ $contentHandlers } ) { + print $contentHandlers->[$contentIndex]."\n"; + ##Insert before Operation + if($contentHandlers->[$contentIndex] eq 'WebGUI::Content::Operation') { + splice @{ $contentHandlers }, $contentIndex, 0, 'WebGUI::Content::PassiveAnalytics'; + last HANDLER; + } + ++$contentIndex; + } + $session->config->set('contentHandlers', $contentHandlers); + } + # Workflow Activities + my $workflowActivities = $session->config->get('workflowActivities'); + my @none = @{ $workflowActivities->{'None'} }; + if (!isIn('WebGUI::Workflow::Activity::SummarizePassiveAnalytics', @none)) { + push @none, 'WebGUI::Workflow::Activity::SummarizePassiveAnalytics'; + } + if (!isIn('WebGUI::Workflow::Activity::BucketPassiveAnalytics', @none)) { + push @none, 'WebGUI::Workflow::Activity::BucketPassiveAnalytics'; + } + $workflowActivities->{'None'} = [ @none ]; + $session->config->set('workflowActivities', $workflowActivities); + print "DONE!\n"; +} + +#---------------------------------------------------------------------------- +# Add the Passive Analytics Workflow +sub pa_installWorkflow { + my $session = shift; + print "\tAdd Passive Analytics Workflow... "; + my $workflow = WebGUI::Workflow->create( + $session, + { + title => 'Analyze Passive Analytics', + mode => 'singleton', + type => 'None', + description => 'Manual changes to this workflow will be lost. Please only use the Passive Analytics screen to make changes', + }, + 'PassiveAnalytics000001', + ); + my $summarize = $workflow->addActivity('WebGUI::Workflow::Activity::SummarizePassiveAnalytics'); + my $bucket = $workflow->addActivity('WebGUI::Workflow::Activity::BucketPassiveAnalytics'); + $summarize->set('title', 'Perform duration analysis'); + $bucket->set( 'title', 'Please log entries into buckets'); + $workflow->set({enabled => 1}); + print "DONE!\n"; +} + #---------------------------------------------------------------------------- # Describe what our function does diff --git a/lib/WebGUI/Content/Asset.pm b/lib/WebGUI/Content/Asset.pm index 3d5e6ba74..64480777a 100644 --- a/lib/WebGUI/Content/Asset.pm +++ b/lib/WebGUI/Content/Asset.pm @@ -18,6 +18,7 @@ use strict; use LWP::MediaTypes qw(guess_media_type); use Time::HiRes; use WebGUI::Asset; +use WebGUI::PassiveAnalytics::Logging; use Apache2::Const -compile => qw(OK); @@ -170,6 +171,9 @@ sub page { $method = "view"; } } + ##Passive Analytics Logging + WebGUI::PassiveAnalytics::Logging::log($session, $asset); + $output = tryAssetMethod($session,$asset,$method); $output = tryAssetMethod($session,$asset,"view") unless ($output || ($method eq "view")); } diff --git a/lib/WebGUI/Content/PassiveAnalytics.pm b/lib/WebGUI/Content/PassiveAnalytics.pm new file mode 100644 index 000000000..7de3b86a8 --- /dev/null +++ b/lib/WebGUI/Content/PassiveAnalytics.pm @@ -0,0 +1,49 @@ +package WebGUI::Content::PassiveAnalytics; + +use strict; +use WebGUI::AdminConsole; +use WebGUI::Exception; +use WebGUI::PassiveAnalytics::Flow; + +=head1 NAME + +Package WebGUI::Content::PassiveAnalytics + +=head1 DESCRIPTION + +Handle all requests for building and editing Passive Analytic flows. + +=head1 SYNOPSIS + + use WebGUI::Content::PassiveAnalytics; + my $output = WebGUI::Content::PassiveAnalytics::handler($session); + +=head1 SUBROUTINES + +These subroutines are available from this package: + +=cut + +#------------------------------------------------------------------- + +=head2 handler ( session ) + +The content handler for this package. + +=cut + +sub handler { + my ($session) = @_; + my $output = undef; + return undef unless $session->form->get('op') eq 'passiveAnalytics'; + my $function = "www_".$session->form->get('func'); + if ($function ne "www_" && (my $sub = WebGUI::PassiveAnalytics::Flow->can($function))) { + $output = $sub->($session); + } + else { + WebGUI::Error::MethodNotFound->throw(error=>"Couldn't call non-existant method $function inside PassiveAnalytics", method=>$function); + } + return $output; +} + +1; diff --git a/lib/WebGUI/PassiveAnalytics/Flow.pm b/lib/WebGUI/PassiveAnalytics/Flow.pm new file mode 100644 index 000000000..6c9538d77 --- /dev/null +++ b/lib/WebGUI/PassiveAnalytics/Flow.pm @@ -0,0 +1,458 @@ +package WebGUI::PassiveAnalytics::Flow; + +use strict; +use Tie::IxHash; +use WebGUI::AdminConsole; +use WebGUI::HTMLForm; +use WebGUI::International; +use WebGUI::Pluggable; +use WebGUI::PassiveAnalytics::Rule; +use WebGUI::Utility; +use WebGUI::HTMLForm; +use WebGUI::Workflow; +use WebGUI::Workflow::Instance; +use WebGUI::User; +use WebGUI::Text; + +=head1 NAME + +Package WebGUI::PassiveAnalytics::Flow + +=head1 DESCRIPTION + +Web interface for making sets of rules for doing passive analytics, and +running them. + +=cut + +#---------------------------------------------------------------------------- + +=head2 analysisActive ( session ) + +Returns true if an instance of the PassiveAnalytics workflow is active. + +=cut + +sub analysisActive { + my $session = shift; + my ($running, $startDate, $endDate, $userId) = $session->db->quickArray(q!select running, startDate, endDate, userId from passiveAnalyticsStatus!); + if (wantarray) { + return $running, $startDate, $endDate, WebGUI::User->new($session, $userId); + } + return $running; +} + +#---------------------------------------------------------------------------- + +=head2 canView ( session [, user] ) + +Returns true if the user can administrate this operation. user defaults to +the current user. + +=cut + +sub canView { + my $session = shift; + my $user = shift || $session->user; + return $user->isInGroup( 3 ); +} + +#---------------------------------------------------------------------------- + +=head2 exportSomething ( session, sth, filename ) + +Generates CSV data from the supplied statement handle and generates +a temporary WebGUI::Storage object containing that data in the requested +filename. + +This subroutine also does a setRedirect to the URL of the file in +the storage object. + +=head3 session + +Session variable, to set the http redirect correctly. + +=head3 sth + +Statement handle for reading data and getting column names + +=head3 filename + +The name of the file to create inside the storage object. + +=cut + +sub exportSomething { + my ($session, $sth, $filename) = @_; + my $storage = WebGUI::Storage->createTemp($session); + my @columns = $sth->getColumnNames; + my $csvData = WebGUI::Text::joinCSV( @columns ). "\n"; + while (my $row = $sth->hashRef()) { + my @row = @{ $row }{@columns}; + $csvData .= WebGUI::Text::joinCSV(@row) . "\n"; + } + $storage->addFileFromScalar($filename, $csvData); + $session->http->setRedirect($storage->getUrl($filename)); +} + +#------------------------------------------------------------------- + +=head2 www_deleteRule ( ) + +Deletes an activity from a workflow. + +=cut + +sub www_deleteRule { + my $session = shift; + return $session->privilege->insufficient() unless canView($session); + my $rule = WebGUI::PassiveAnalytics::Rule->new($session, $session->form->get("ruleId")); + if (defined $rule) { + $rule->delete; + } + return www_editRuleflow($session); +} + +#------------------------------------------------------------------ + +=head2 www_demoteRule ( session ) + +Moves a Rule down one position in the execution order. + +=head3 session + +A reference to the current session. + +=cut + +sub www_demoteRule { + my $session = shift; + return $session->privilege->insufficient() unless canView($session); + my $rule = WebGUI::PassiveAnalytics::Rule->new($session, $session->form->get("ruleId")); + if (defined $rule) { + $rule->demote; + } + return www_editRuleflow($session); +} + +#------------------------------------------------------------------- + +=head2 www_editRuleflow ( session ) + +Configure a set of analyses to run on the passive logs. The analysis is destructive. + +=cut + +sub www_editRuleflow { + my $session = shift; + my $error = shift; + return $session->privilege->insufficient() unless canView($session); + my ($running, $startDate, $endDate, $user) = analysisActive($session); + if ($error) { + $error = qq|
$error
\n|; + } + elsif (!$running) { + $error = qq|
Passive Analytics analysis completed on $endDate
\n|; + } + my $i18n = WebGUI::International->new($session, "PassiveAnalytics"); + my $addmenu = '
'; + $addmenu .= sprintf '%s', + $session->url->page('op=passiveAnalytics;func=editRule'), + $i18n->get('Add a bucket'); + $addmenu .= '
'; + my $f = WebGUI::HTMLForm->new($session); + $f->hidden( + name=>'op', + value=>'passiveAnalytics' + ); + $f->hidden( + name=>'func', + value=>'editRuleflowSave' + ); + $f->integer( + name => 'pauseInterval', + value => $session->form->get('pauseInterval') || $session->setting->get('passiveAnalyticsInterval') || 300, + label => $i18n->get('pause interval'), + hoverHelp => $i18n->get('pause interval help'), + ); + if ($running) { + $f->raw(sprintf <username); +Passive Analytics analysis is currently active. Analysis was begun at %s by %s +EOD + } + else { + $f->submit(value => $i18n->get('Begin analysis')); + } + my $steps = ''; + my $getARule = WebGUI::PassiveAnalytics::Rule->getAllIterator($session); + my $icon = $session->icon; + while (my $rule = $getARule->()) { + my $id = $rule->getId; + my $bucket = $rule->get('bucketName'); + $steps .= ''; + + } + $steps .= '
' + . $icon->delete( 'op=passiveAnalytics;func=deleteRule;ruleId='.$id, undef, $i18n->get('confirm delete rule')) + . $icon->edit( 'op=passiveAnalytics;func=editRule;ruleId='.$id) + . $icon->moveDown('op=passiveAnalytics;func=demoteRule;ruleId='.$id) + . $icon->moveUp( 'op=passiveAnalytics;func=promoteRule;ruleId='.$id) + . ''.$bucket.'
 Other
'; + my $ac = WebGUI::AdminConsole->new($session,'passiveAnalytics'); + $ac->addSubmenuItem($session->url->page('op=passiveAnalytics;func=settings'), $i18n->get('Passive Analytics Settings')); + if (!$running) { + $ac->addSubmenuItem($session->url->page('op=passiveAnalytics;func=exportBucketData'), $i18n->get('Export bucket data')); + $ac->addSubmenuItem($session->url->page('op=passiveAnalytics;func=exportDeltaData'), $i18n->get('Export delta data')); + $ac->addSubmenuItem($session->url->page('op=passiveAnalytics;func=exportLogs'), $i18n->get('Export raw logs')); + } + return $ac->render($error.$f->print.$addmenu.$steps, 'Passive Analytics'); +} + +#------------------------------------------------------------------- + +=head2 www_editRuleflowSave ( ) + +Saves the results of www_editRuleflow() + +=cut + +sub www_editRuleflowSave { + my $session = shift; + return $session->privilege->insufficient() unless canView($session); + return www_editRuleflow($session, 'Passive Analytics is already active. Please do not try to subvert the UI in the future') + if analysisActive($session); + my $workflow = WebGUI::Workflow->new($session, 'PassiveAnalytics000001'); + return www_editRuleflow($session, "The Passive Analytics workflow has been deleted. Please contact an Administrator immediately.") unless defined $workflow; + my $delta = $session->form->process('pauseInterval','integer'); + my $activities = $workflow->getActivities(); + ##Note, they're in order, and the order is known. + $activities->[0]->set('deltaInterval', $delta); + $activities->[1]->set('userId', $session->user->userId); + my $instance = WebGUI::Workflow::Instance->create($session, { + workflowId => $workflow->getId, + priority => 1, + }); + if (!defined $instance) { + return www_editRuleflow($session, "A Passive Analytics analysis is currently running.") if $session->stow->get('singletonWorkflowClash'); + return www_editRuleflow($session, "Error creating the workflow instance."); + } + $instance->start('skipRealtime'); + $session->db->write('update passiveAnalyticsStatus set startDate=NOW(), userId=?, endDate=?, running=1', [$session->user->userId, '']); + return www_editRuleflow($session); +} + + +#------------------------------------------------------------------- + +=head2 www_editRule ( ) + +Displays a form to edit the properties rule. + +=cut + +sub www_editRule { + my ($session, $error) = @_; + return $session->privilege->insufficient() unless canView($session); + + if ($error) { + $error = qq|
$error
\n|; + } + ##Make a PassiveAnalytics rule to use to populate the form. + my $ruleId = $session->form->get('ruleId'); + my $rule; + if ($ruleId) { + $rule = WebGUI::PassiveAnalytics::Rule->new($session, $ruleId); + } + else { + ##We need a temporary rule so that we can call dynamicForm, below + $ruleId = 'new'; + $rule = WebGUI::PassiveAnalytics::Rule->create($session, {}); + } + + ##Build the form + my $form = WebGUI::HTMLForm->new($session); + $form->hidden( name=>"op", value=>"passiveAnalytics"); + $form->hidden( name=>"func", value=>"editRuleSave"); + $form->hidden( name=>"ruleId", value=>$ruleId); + $form->dynamicForm([WebGUI::PassiveAnalytics::Rule->crud_definition($session)], 'properties', $rule); + $form->submit; + + my $i18n = WebGUI::International->new($session, 'PassiveAnalytics'); + my $ac = WebGUI::AdminConsole->new($session,'passiveAnalytics'); + $ac->addSubmenuItem($session->url->page("op=passiveAnalytics;func=editRuleflow"), $i18n->get("manage ruleset")); + if ($ruleId eq 'new') { + $rule->delete; + } + return $ac->render($error.$form->print,$i18n->get('Edit Rule')); +} + +#------------------------------------------------------------------- + +=head2 www_editRuleSave ( ) + +Saves the results of www_editRule(). + +=cut + +sub www_editRuleSave { + my $session = shift; + my $form = $session->form; + return $session->privilege->insufficient() unless canView($session); + my $regexp = $form->get('regexp'); + eval { + 'fooBarBaz' =~ qr/$regexp/; + }; + if ($@) { + my $error = $@; + $error =~ s/at \S+?\.pm line \d+.*$//; + my $i18n = WebGUI::International->new($session, 'PassiveAnalytics'); + $error = join ' ', $i18n->get('Regular Expression Error:'), $error; + return www_editRule($session, $error); + } + my $ruleId = $form->get('ruleId'); + my $rule; + if ($ruleId eq 'new') { + $rule = WebGUI::PassiveAnalytics::Rule->create($session, {}); + } + else { + $rule = WebGUI::PassiveAnalytics::Rule->new($session, $ruleId); + } + $rule->updateFromFormPost if $rule; + return www_editRuleflow($session); +} + +#------------------------------------------------------------------- + +=head2 www_exportBucketData ( ) + +Dump the contents of the bucket log. + +=cut + +sub www_exportBucketData { + my ($session) = @_; + my $bucket = $session->db->read('select * from bucketLog order by userId, Bucket, timeStamp'); + exportSomething($session, $bucket, 'bucketData.csv'); + return "redirect"; +} + +#------------------------------------------------------------------- + +=head2 www_exportDeltaData ( ) + +Dump the contents of the delta log. + +=cut + +sub www_exportDeltaData { + my ($session) = @_; + my $delta = $session->db->read('select * from deltaLog order by userId, timeStamp'); + exportSomething($session, $delta, 'deltaData.csv'); + return "redirect"; +} + +#------------------------------------------------------------------- + +=head2 www_exportLogs ( ) + +Dump the contents of the raw log. + +=cut + +sub www_exportLogs { + my ($session) = @_; + my $raw = $session->db->read('select * from passiveLog order by userId, timeStamp'); + exportSomething($session, $raw, 'passiveData.csv'); + return "redirect"; +} + +#------------------------------------------------------------------ + +=head2 www_promoteRule ( session ) + +Moves a rule up one position in the execution order. + +=head3 session + +A reference to the current session. + +=cut + +sub www_promoteRule { + my $session = shift; + return $session->privilege->insufficient() unless canView($session); + my $rule = WebGUI::PassiveAnalytics::Rule->new($session, $session->form->get("ruleId")); + if (defined $rule) { + $rule->promote; + } + return www_editRuleflow($session); +} + +#------------------------------------------------------------------- + +=head2 www_settings ( session ) + +Configure Passive Analytics settings. + +=cut + +sub www_settings { + my $session = shift; + my $error = shift; + return $session->privilege->insufficient() unless canView($session); + if ($error) { + $error = qq|
$error
\n|; + } + my $i18n = WebGUI::International->new($session, "PassiveAnalytics"); + my $f = WebGUI::HTMLForm->new($session); + $f->hidden( + name=>'op', + value=>'passiveAnalytics' + ); + $f->hidden( + name=>'func', + value=>'settingsSave' + ); + $f->integer( + name => 'pauseInterval', + value => $session->form->get('pauseInterval') || $session->setting->get('passiveAnalyticsInterval') || 300, + label => $i18n->get('default pause interval'), + hoverHelp => $i18n->get('default pause interval help'), + ); + $f->yesNo( + name => 'deleteDelta', + value => $session->form->get('deleteDelta') || $session->setting->get('passiveAnalyticsDeleteDelta') || 0, + label => $i18n->get('Delete Delta Table?'), + hoverHelp => $i18n->get('Delete Delta Table? help'), + ); + $f->yesNo( + name => 'enabled', + value => $session->form->get('enabled') || $session->setting->get('passiveAnalyticsEnabled') || 0, + label => $i18n->get('Enabled?'), + hoverHelp => $i18n->get('Enabled? help'), + ); + $f->submit(); + my $ac = WebGUI::AdminConsole->new($session,'passiveAnalytics'); + $ac->addSubmenuItem($session->url->page('op=passiveAnalytics;func=editRuleflow'), $i18n->get('Passive Analytics')); + return $ac->render($error.$f->print, 'Passive Analytics Settings'); +} + +#------------------------------------------------------------------- + +=head2 www_settingsSave ( session ) + +Save Passive Analytics settings. + +=cut + +sub www_settingsSave { + my $session = shift; + return $session->privilege->insufficient() unless canView($session); + my $form = $session->form; + $session->setting->set('passiveAnalyticsInterval', $form->process('pauseInterval', 'integer')); + $session->setting->set('passiveAnalyticsDeleteDelta', $form->process('deleteDelta', 'yesNo' )); + $session->setting->set('passiveAnalyticsEnabled', $form->process('enabled', 'yesNo' )); + return www_settings($session); +} + +1; diff --git a/lib/WebGUI/PassiveAnalytics/Logging.pm b/lib/WebGUI/PassiveAnalytics/Logging.pm new file mode 100644 index 000000000..26f5bd3d8 --- /dev/null +++ b/lib/WebGUI/PassiveAnalytics/Logging.pm @@ -0,0 +1,47 @@ +package WebGUI::PassiveAnalytics::Logging; + +use strict; +use WebGUI::Session; +use WebGUI::Asset; + +=head1 NAME + +Package WebGUI::PassiveAnalytics::Logging + +=head1 DESCRIPTION + +Encapsulate all logging functions in here. + +=cut + +#---------------------------------------------------------------------------- + +=head2 log ( session, asset ) + +Log Passive Analytics data to the db. + +=head3 session + +A session variable. + +=head3 asset + +The asset to log. + +=cut + +sub log { + my ($session, $asset) = @_; + return unless $session->setting->get('passiveAnalyticsEnabled'); + my $assetClass = $asset->get('className'); + $assetClass =~ s/^WebGUI::Asset:://; + if ( $assetClass ne 'Snippet' + && substr($assetClass,0,4) ne 'File') { + $session->db->write( + q|INSERT INTO `passiveLog` (userId, sessionId, assetId, timestamp, url) VALUES (?,?,?,?,?)|, + [ $session->user->userId, $session->getId, $asset->getId, time(), $session->request->unparsed_uri,] + ); + } +} + +1; diff --git a/lib/WebGUI/PassiveAnalytics/Rule.pm b/lib/WebGUI/PassiveAnalytics/Rule.pm new file mode 100644 index 000000000..5d052c8da --- /dev/null +++ b/lib/WebGUI/PassiveAnalytics/Rule.pm @@ -0,0 +1,92 @@ +package WebGUI::PassiveAnalytics::Rule; + +use base qw/WebGUI::Crud/; +use WebGUI::International; + +=head1 NAME + +Package WebGUI::PassiveAnalytics::Rule; + +=head1 DESCRIPTION + +Base class for rules that are used to analyze the Passive Analytics log. + +=head1 METHODS + +These methods are available from this class: + +=cut + +#------------------------------------------------------------------- + +=head2 crud_definition ( ) + +WebGUI::Crud definition for this class. + +=head3 tableName + +analyticRule. + +=head3 tableKey + +ruleId + +=head3 sequenceKey + +None. There is only 1 sequence of rules for a site. + +=head3 properties + +=head4 bucketName + +The name of a bucket to hold results for this rule. + +=head4 rules + +JSON blob with configuration data for the individual rules. + +=cut + +sub crud_definition { + my ($class, $session) = @_; + my $definition = $class->SUPER::crud_definition($session); + $definition->{tableName} = 'analyticRule'; + $definition->{tableKey} = 'ruleId'; + $definition->{sequenceKey} = ''; + my $properties = $definition->{properties}; + my $i18n = WebGUI::International->new($session); + $properties->{bucketName} = { + fieldType => 'text', + label => $i18n->get('Bucket Name','PassiveAnalytics'), + hoverHelp => $i18n->get('Bucket Name help','PassiveAnalytics'), + defaultValue => '', + }; + $properties->{regexp} = { + fieldType => 'text', + label => $i18n->get('regexp','PassiveAnalytics'), + hoverHelp => $i18n->get('regexp help','PassiveAnalytics'), + defaultValue => '.+', + }; + return $definition; +} + +#------------------------------------------------------------------- + +=head2 matchesBucket ( $logLine ) + +Executes the rule to determine if a log file entry matches the rule. + +=head3 $logLine + +A hashref of information from 1 line of the logs. + +=cut + +sub matchesBucket { + my ($self, $logLine) = @_; + my $regexp = $self->get('regexp'); + return $logLine->{url} =~ m/$regexp/; +} + +1; +#vim:ft=perl diff --git a/lib/WebGUI/Workflow/Activity/BucketPassiveAnalytics.pm b/lib/WebGUI/Workflow/Activity/BucketPassiveAnalytics.pm new file mode 100644 index 000000000..56cb8e1f8 --- /dev/null +++ b/lib/WebGUI/Workflow/Activity/BucketPassiveAnalytics.pm @@ -0,0 +1,153 @@ +package WebGUI::Workflow::Activity::BucketPassiveAnalytics; + +use strict; +use base 'WebGUI::Workflow::Activity'; +use WebGUI::PassiveAnalytics::Rule; +use WebGUI::Inbox; + +=head1 NAME + +Package WebGUI::Workflow::Activity::BucketPassiveAnalytics + +=head1 DESCRIPTION + +Run through a set of rules to figure out how to classify log file entries. + +=head1 SYNOPSIS + +See WebGUI::Workflow::Activity for details on how to use any activity. + +=head1 METHODS + +These methods are available from this class: + +=cut + + +#------------------------------------------------------------------- + +=head2 definition ( session, definition ) + +See WebGUI::Workflow::Activity::defintion() for details. + +=cut + +sub definition { + my $class = shift; + my $session = shift; + my $definition = shift; + my $i18n = WebGUI::International->new($session, "PassiveAnalytics"); + push( @{$definition}, { + name=>$i18n->get("Bucket Passive Analytics"), + properties=> { + notifyUser => { + fieldType => 'user', + label => $i18n->get('User'), + hoverHelp => $i18n->get('User help'), + defaultValue => $session->user->userId, + }, + }, + }); + return $class->SUPER::definition($session,$definition); +} + + +#------------------------------------------------------------------- + +=head2 execute ( [ object ] ) + +Analyze the deltaLog table, and generate the bucketLog table. + +=head3 notes + +=cut + +sub execute { + my ($self, undef, $instance) = @_; + my $session = $self->session; + my $endTime = time() + $self->getTTL; + my $expired = 0; + + ##Load all the rules into an array + my @rules = (); + my $getARule = WebGUI::PassiveAnalytics::Rule->getAllIterator($session); + while (my $rule = $getARule->()) { + my $regexp = $rule->get('regexp'); + push @rules, [ $rule->get('bucketName'), qr/$regexp/]; + } + + ##Get the index stored from the last invocation of the Activity. If this is + ##the first run, then clear out the table. + my $logIndex = $instance->getScratch('lastPassiveLogIndex') || 0; + if ($logIndex == 0) { + $session->db->write('delete from bucketLog'); + } + my %bucketCache = (); + + ##Configure all the SQL + my $deltaSql = <<"EOSQL1"; +select userId, assetId, url, delta, from_unixtime(timeStamp) as stamp + from deltaLog order by timestamp limit $logIndex, 1234567890 +EOSQL1 + my $deltaSth = $session->db->read($deltaSql); + my $bucketSth = $session->db->prepare('insert into bucketLog (userId, Bucket, duration, timeStamp) VALUES (?,?,?,?)'); + + ##Walk through the log file entries, one by one. Run each entry against + ##all the rules until 1 matches. If it doesn't match any rule, then bin it + ##into the "Other" bucket. + DELTA_ENTRY: while (my $entry = $deltaSth->hashRef()) { + ++$logIndex; + my $bucketFound = 0; + my $url = $entry->{url}; + if (exists $bucketCache{$url}) { + $bucketSth->execute([$entry->{userId}, $bucketCache{$url}, $entry->{delta}, $entry->{stamp}]); + } + else { + RULE: foreach my $rule (@rules) { + next RULE unless $url =~ $rule->[1]; + + # Into the bucket she goes.. + $bucketCache{$url} = $rule->[0]; + $bucketSth->execute([$entry->{userId}, $rule->[0], $entry->{delta}, $entry->{stamp}]); + $bucketFound = 1; + last RULE; + } + if (!$bucketFound) { + $bucketCache{$url} = 'Other'; + $bucketSth->execute([$entry->{userId}, 'Other', $entry->{delta}, $entry->{stamp}]); + } + } + if (time() > $endTime) { + $expired = 1; + last DELTA_ENTRY; + } + } + + if ($expired) { + $instance->setScratch('logIndex', $logIndex); + return $self->WAITING(1); + } + my $message = 'Passive analytics is done.'; + if ($session->setting->get('passiveAnalyticsDeleteDelta')) { + $session->log->info('Clearing Passive Analytics delta log'); + $session->db->write('delete from deltaLog'); + $message .= ' The delta log has been cleaned up.'; + } + ##If userId was set to 0, do not send any emails. + if ($self->get('userId')) { + my $inbox = WebGUI::Inbox->new($self->session); + $inbox->addMessage({ + status => 'unread', + subject => 'Passive analytics is done', + userId => $self->get('userId'), + message => $message, + }); + } + $session->db->write('update passiveAnalyticsStatus set endDate=NOW(), running=0'); + + return $self->COMPLETE; +} + +1; + +#vim:ft=perl diff --git a/lib/WebGUI/Workflow/Activity/SummarizePassiveAnalytics.pm b/lib/WebGUI/Workflow/Activity/SummarizePassiveAnalytics.pm new file mode 100644 index 000000000..6b4c2bcde --- /dev/null +++ b/lib/WebGUI/Workflow/Activity/SummarizePassiveAnalytics.pm @@ -0,0 +1,148 @@ +package WebGUI::Workflow::Activity::SummarizePassiveAnalytics; + +use strict; +use base 'WebGUI::Workflow::Activity'; + +=head1 NAME + +Package WebGUI::Workflow::Activity::SummarizePassiveAnalytics + +=head1 DESCRIPTION + +Summarize how long a user stayed on a page, using a user supplied interval. + +=head1 SYNOPSIS + +See WebGUI::Workflow::Activity for details on how to use any activity. + +=head1 METHODS + +These methods are available from this class: + +=cut + + +#------------------------------------------------------------------- + +=head2 definition ( session, definition ) + +See WebGUI::Workflow::Activity::defintion() for details. + +=cut + +sub definition { + my $class = shift; + my $session = shift; + my $definition = shift; + my $i18n = WebGUI::International->new($session, 'PassiveAnalytics'); + push(@{$definition}, { + name=>$i18n->get('Summarize Passive Analytics'), + properties=> { + deltaInterval => { + fieldType => 'interval', + label => $i18n->get('pause interval'), + defaultValue => 15, + hoverHelp => $i18n->get('pause interval help'), + }, + } + }); + return $class->SUPER::definition($session,$definition); +} + + +#------------------------------------------------------------------- + +=head2 execute ( [ object ] ) + +Analyze the passiveLog table, and generate the deltaLog table. + +=head3 notes + +If there is only 1 line in the table for a particular sessionId or +userId, no conclusions as to how long the user viewed a page can be +drawn from that. Similarly, the last entry in their browsing log +yields no data, since we require another entry in the passiveLog to +determine a delta. + +=cut + +sub execute { + my ($self, undef, $instance) = @_; + my $session = $self->session; + my $endTime = time() + $self->getTTL; + my $deltaInterval = $self->get('deltaInterval'); + + my $passive = q{select * from passiveLog where userId <> '1' order by userId, sessionId, timeStamp}; + my $sth; + my $lastUserId; + my $lastSessionId; + my $lastTimeStamp; + my $lastAssetId; + my $lastUrl; + my $counter = $instance->getScratch('counter'); + if ($counter) { + $passive .= ' limit '. $counter .', 1234567890'; + $sth = $session->db->read($passive); + $lastUserId = $instance->getScratch('lastUserId'); + $lastSessionId = $instance->getScratch('lastSessionId'); + $lastTimeStamp = $instance->getScratch('lastTimeStamp'); + $lastAssetId = $instance->getScratch('lastAssetId'); + $lastUrl = $instance->getScratch('lastUrl'); + } + else { + $sth = $session->db->read($passive); + my $logLine = $sth->hashRef(); + $lastUserId = $logLine->{userId}; + $lastSessionId = $logLine->{sessionId}; + $lastTimeStamp = $logLine->{timeStamp}; + $lastAssetId = $logLine->{assetId}; + $lastUrl = $logLine->{url}; + } + + $session->db->write('delete from deltaLog'); ##Only if we're starting out + my $deltaLog = $session->db->prepare('insert into deltaLog (userId, assetId, delta, timeStamp, url) VALUES (?,?,?,?,?)'); + + my $expired = 0; + LOG_ENTRY: while (my $logLine = $sth->hashRef()) { + $counter++; + my $delta = $logLine->{timeStamp} - $lastTimeStamp; + if ( $logLine->{userId} eq $lastUserId + && $logLine->{sessionId} eq $lastSessionId + && $delta < $deltaInterval ) { + $deltaLog->execute([$lastUserId, $lastAssetId, $delta, $lastTimeStamp, $lastUrl]); + } + $lastUserId = $logLine->{userId}; + $lastSessionId = $logLine->{sessionId}; + $lastTimeStamp = $logLine->{timeStamp}; + $lastAssetId = $logLine->{assetId}; + $lastUrl = $logLine->{url}; + if (time() > $endTime) { + $instance->setScratch('lastUserId', $lastUserId); + $instance->setScratch('lastSessionId', $lastSessionId); + $instance->setScratch('lastTimeStamp', $lastTimeStamp); + $instance->setScratch('lastAssetId', $lastAssetId); + $instance->setScratch('lastUrl', $lastUrl); + $instance->setScratch('counter', $counter); + $expired = 1; + last LOG_ENTRY; + } + } + + if ($expired) { + return $self->WAITING(1); + } + + $instance->deleteScratch('lastUserId'); + $instance->deleteScratch('lastSessionId'); + $instance->deleteScratch('lastTimeStamp'); + $instance->deleteScratch('lastAssetId'); + $instance->deleteScratch('lastUrl'); + $instance->deleteScratch('counter'); + return $self->COMPLETE; +} + + + +1; + +#vim:ft=perl diff --git a/lib/WebGUI/i18n/English/PassiveAnalytics.pm b/lib/WebGUI/i18n/English/PassiveAnalytics.pm new file mode 100644 index 000000000..4cca51390 --- /dev/null +++ b/lib/WebGUI/i18n/English/PassiveAnalytics.pm @@ -0,0 +1,165 @@ +package WebGUI::i18n::English::PassiveAnalytics; + +use strict; + +our $I18N = { + + 'Summarize Passive Analytics' => { + message => q|Summarize Passive Analytics|, + context => q|The name of this workflow activity.|, + lastUpdated => 0, + }, + + 'pause interval' => { + message => q|Pause threshold|, + lastUpdated => 0, + }, + + 'pause interval help' => { + message => q|Set the time between clicks that is interpreted as the user reading the page, as opposed to beginning a new browsing session, or leaving the site.|, + lastUpdated => 0, + }, + + 'default pause interval' => { + message => q|Default Pause Threshold|, + lastUpdated => 0, + }, + + 'default pause interval help' => { + message => q|Set the default pause interval displayed the user sees in the Passive Analytics screen.|, + lastUpdated => 0, + }, + + 'other' => { + message => q|Other|, + lastUpdated => 0, + context => q|Meaning not like anything in a set. This, that and the other one. Also, a catch all.| + }, + + 'Bucket Name' => { + message => q|Bucket Name|, + lastUpdated => 0, + context => q|To name a container, or bucket.| + }, + + 'Bucket Name help' => { + message => q|Pick a unique, descriptive short name for this bucket.|, + lastUpdated => 0, + context => q|| + }, + + 'regexp' => { + message => q|Regular expression|, + lastUpdated => 0, + context => q|| + }, + + 'regexp help' => { + message => q|Define a regular expression to pick log entries for this bucket.
+^ = beginning of url
+$ = end of url
+. = any character
+* = any amount
++ = 1 or more
+? = 0 or 1
+Meta characters should be backslash-escaped if you want to match them as ordinary text, e.g.
+home\?func=match, or
+|, + lastUpdated => 0, + context => q|| + }, + + 'Passive Analytics' => { + message => q|Passive Analytics|, + lastUpdated => 0, + context => q|| + }, + + 'Passive Analytics Settings' => { + message => q|Passive Analytics Settings|, + lastUpdated => 0, + context => q|| + }, + + 'Edit Rule' => { + message => q|Edit Rule|, + lastUpdated => 0, + context => q|| + }, + + 'Add a bucket' => { + message => q|Add a bucket|, + lastUpdated => 0, + context => q|| + }, + + 'User' => { + message => q|User|, + lastUpdated => 0, + context => q|| + }, + + 'User help' => { + message => q|The user who will recieve an email when bucket processing is done.|, + lastUpdated => 0, + context => q|| + }, + + 'Begin analysis' => { + message => q|Begin analysis|, + lastUpdated => 0, + context => q|Button label to begin analyzing the logs.| + }, + + 'Delete Delta Table?' => { + message => q|Delete Delta Table?|, + lastUpdated => 0, + context => q|Button label to begin analyzing the logs.| + }, + + 'Delete Delta Table? help' => { + message => q|Should the delta table be cleaned up after the Passive Analytics analyzer is done?|, + lastUpdated => 0, + context => q|Button label to begin analyzing the logs.| + }, + + 'Enabled?' => { + message => q|Enable Passive Analytics?|, + lastUpdated => 0, + context => q|| + }, + + 'Enabled? help' => { + message => q|Passive Analytics will do no logging until enabled.|, + lastUpdated => 0, + context => q|| + }, + + 'Regular Expression Error:' => { + message => q|Regular Expression Error:|, + lastUpdated => 0, + context => q|Error displayed when a user enters in a bad regular expression. This label will be followed by the error from perl.| + }, + + 'Export bucket data' => { + message => q|Export bucket data|, + lastUpdated => 0, + context => q|URL label to export data in CSV format|, + }, + + 'Export delta data' => { + message => q|Export delta data|, + lastUpdated => 0, + context => q|URL label to export data in CSV format|, + }, + + 'Export raw logs' => { + message => q|Export raw logs|, + lastUpdated => 0, + context => q|URL label to raw log data in CSV format|, + }, + +}; + +1; +#vim:ft=perl diff --git a/t/Workflow/Activity/BucketPassiveAnalytics.t b/t/Workflow/Activity/BucketPassiveAnalytics.t new file mode 100644 index 000000000..b05ab5cb4 --- /dev/null +++ b/t/Workflow/Activity/BucketPassiveAnalytics.t @@ -0,0 +1,108 @@ + +use FindBin; +use strict; +use lib "$FindBin::Bin/../../lib"; +#use DB; + +use WebGUI::Test; +use WebGUI::Asset; +use WebGUI::PassiveAnalytics::Rule; +use WebGUI::Workflow::Activity::BucketPassiveAnalytics; +use WebGUI::Text; + +use Test::More; + +plan tests => 1; # increment this value for each test you create + +my $session = WebGUI::Test->session; +$session->user({userId => 3}); + +my $workflow = WebGUI::Workflow->new($session, 'PassiveAnalytics000001'); +my $activities = $workflow->getActivities(); +##Note, they're in order, and the order is known. +$activities->[0]->set('deltaInterval', 100); +$activities->[1]->set('userId', 0); ##To disable sending emails +diag "Configured activities"; + +my $instance = WebGUI::Workflow::Instance->create($session, + { + workflowId => $workflow->getId, + skipSpectreNotification => 1, + priority => 1, + } +); +##Rule label, url, and regexp +my @ruleSets = ( + ['home', '/home', '^\/home' ], + ['one', '/one', '^\/one$' ], + ['two', '/two', '^\/two$' ], + ['three', '/three', '^\/three$' ], + ['end', '/blah/blah/end', 'end$' ], + ['casa', '/home/casa', 'casa$' ], + ['uno', '/one/uno', 'uno$' ], + ['dos', '/two/dos', 'dos$' ], + ['tres', '/three/tres', 'tres$' ], + ['alpha', '/alpha/aee', '.alpha.aee' ], + ['beta', '/beta/bee', '.beta.bee' ], + ['gamma', '/gamma/cee', '.gamma.cee' ], + ['delta', '/delta/dee', '.delta.dee' ], + ['eee', '/epsilon/eee', 'eee$' ], + ['thingy1', '/thingy?thingId=1', '^.thingy\?thingId=1' ], + ['rogerRoger', '/roger/roger', '(?:\/roger){2}' ], + ['roger', '/roger', '^\/roger' ], + ['thingy2', '/thingy?thingId=2', '^.thingy\?thingId=2' ], + ['beet', '/beta/beet', '.beta.beet' ], + ['zero', '/yelnats', 'yelnats' ], +); + +my @url2 = @ruleSets; +diag "Making rules"; +while (my $spec = shift @url2) { + my ($bucket, undef, $regexp) = @{ $spec }; + WebGUI::PassiveAnalytics::Rule->create($session, { bucketName => $bucket, regexp => $regexp }); +} + +my @urls = map {$_->[1]} @ruleSets; +diag "Making log data with " . scalar @urls . " urls"; +loadLogData($session, @urls); +diag "Data logged"; + +##Build rulesets + +##Now, run it and wait for it to finish +my $counter = 0; +diag time(); +#DB::enable_profile(); +PAUSE: while (my $retval = $instance->run()) { + diag $retval; + last PAUSE if $retval eq 'done'; + last PAUSE if $counter++ >= 16; +} +#DB::disable_profile(); +diag time(); + +ok(1, 'One test'); + +END { + $session->db->write('delete from passiveLog'); + $session->db->write('delete from analyticRule'); + $instance->delete; +} + +sub loadLogData { + my ($session, @urls) = @_; + $session->db->write('delete from passiveLog'); + my $insert = $session->db->prepare( + q!insert into passiveLog (userId, sessionId, timeStamp, url, assetId) VALUES (?,?,?,?,'assetId')! + ); + my $logCount = 15000; + my $counter; + my $startTime = 1000; + my $numUrls = scalar @urls; + while ($counter++ < $logCount) { + my $index = int rand($numUrls); + my $url = $urls[$index]; + $insert->execute([2, 25, $startTime, $url]); + $startTime += int(rand(10))+1; + } +} diff --git a/www/extras/adminConsole/passiveAnalytics.png b/www/extras/adminConsole/passiveAnalytics.png new file mode 100644 index 000000000..131ebdad8 Binary files /dev/null and b/www/extras/adminConsole/passiveAnalytics.png differ diff --git a/www/extras/adminConsole/small/passiveAnalytics.png b/www/extras/adminConsole/small/passiveAnalytics.png new file mode 100644 index 000000000..a814efd63 Binary files /dev/null and b/www/extras/adminConsole/small/passiveAnalytics.png differ