Attachment #795 for bug #1664

View | Details | Raw Unified | Return to bug 1664
Collapse All | Expand All




  @DBNAMES @DB_EXTENSIONS
  $NSPAM_MAGIC_TOKEN $NHAM_MAGIC_TOKEN $LAST_EXPIRE_MAGIC_TOKEN
  $NTOKENS_MAGIC_TOKEN $OLDEST_TOKEN_AGE_MAGIC_TOKEN
  $SCANCOUNT_BASE_MAGIC_TOKEN $NOW_REBUILDING_MAGIC_TOKEN
};

@ISA = qw();

$LAST_EXPIRE_MAGIC_TOKEN = '**LASTEXPIRE';
$NTOKENS_MAGIC_TOKEN = '**NTOKENS';
$SCANCOUNT_BASE_MAGIC_TOKEN = '**SCANBASE';
$NOW_REBUILDING_MAGIC_TOKEN = '**NOWREBUILDING';

use constant MAX_SIZE_FOR_SCAN_COUNT_FILE => 5000;


	  || $tok eq $LAST_EXPIRE_MAGIC_TOKEN
	  || $tok eq $NTOKENS_MAGIC_TOKEN
	  || $tok eq $OLDEST_TOKEN_AGE_MAGIC_TOKEN
	  || $tok eq $NOW_REBUILDING_MAGIC_TOKEN
	  || $tok eq $SCANCOUNT_BASE_MAGIC_TOKEN);

    my ($ts, $th, $atime) = $self->tok_get ($tok);

      }
    }

    if ((($kept + $deleted) % 1000) == 0) {
      # update "now expiring" token.
      $self->{db_toks}->{$NOW_REBUILDING_MAGIC_TOKEN} = time();
      if ($showdots) { print STDERR "."; }
    }
  }


  $new_toks{$NHAM_MAGIC_TOKEN} = $self->{db_toks}->{$NHAM_MAGIC_TOKEN};
  $new_toks{$NTOKENS_MAGIC_TOKEN} = $kept + $reprieved;

  # ensure this is NOT set in the new db
  delete $new_toks{$NOW_REBUILDING_MAGIC_TOKEN};

  # now untie so we can do renames
  untie %{$self->{db_toks}};
  untie %new_toks;

  # ok, once that's done we can re-tie.  Call untie_db() first so
  # we unlock correctly etc. first
  $self->untie_db();
  $self->tie_db_writable();

  my $done = time();


  if (!$ntoks || $ntoks =~ /\D/) 
		{ $ntoks = $self->{expiry_min_db_size} + 1; }
  if ($ntoks <= $self->{expiry_min_db_size}) {
    dbg ("bayes: expiry not needed, db too small ($ntoks)");
    return 0;
  }

  # is the database currently being expired, or did an expiry attempt
  # fail for some reason in the last 300 seconds?
  my $timenow = time();
  my $amexping = $self->{db_toks}->{$NOW_REBUILDING_MAGIC_TOKEN};
  if (defined ($amexping) && $timenow - $amexping < 300) { # TODO: conf item?
    dbg ("bayes: expiry already in progress ($amexping)");
    return 0;
  }


    return 1;
  }

  dbg ("bayes: expiry not needed, happened recently ($last, now=$now)");
  0;
}


  my $showdots = $opts->{showdots};
  my $retirepath = $path.".old";

  # Set this now.  Unset it at end of expire_old_tokens_trapped().
  # NOTE: we do NOT unset it on error.   If something went
  # wrong, give the server 300 secs to recover in case it was
  # a system-level problem, or caused by load.
  $self->{db_toks}->{$NOW_REBUILDING_MAGIC_TOKEN} = $started;

  # now read the retired journal
  if (!open (JOURNAL, "<$path")) {
    warn "bayes: cannot open read $path\n";

      warn "Bayes journal: gibberish entry found: $_";
    }

    if (($total_count % 1000) == 0) {
      # update "now expiring" token.
      $self->{db_toks}->{$NOW_REBUILDING_MAGIC_TOKEN} = time();
#     if ($showdots) { print STDERR "."; }
    }
  }
  close JOURNAL;


  while( my($k,$v) = each %tokens ) {
    $self->tok_touch_token ($v, $k);

    if ((++$count % 1000) == 0) {
      # update "now expiring" token.
      $self->{db_toks}->{$NOW_REBUILDING_MAGIC_TOKEN} = time();
      if ($showdots) { print STDERR "."; }
    }
  }


Return to bug 1664

Lines 19-25 Link Here

(-)BayesStore.pm.post-793 (-9 / +36 lines)
19	@DBNAMES @DB_EXTENSIONS	19	@DBNAMES @DB_EXTENSIONS
20	$NSPAM_MAGIC_TOKEN $NHAM_MAGIC_TOKEN $LAST_EXPIRE_MAGIC_TOKEN	20	$NSPAM_MAGIC_TOKEN $NHAM_MAGIC_TOKEN $LAST_EXPIRE_MAGIC_TOKEN
21	$NTOKENS_MAGIC_TOKEN $OLDEST_TOKEN_AGE_MAGIC_TOKEN	21	$NTOKENS_MAGIC_TOKEN $OLDEST_TOKEN_AGE_MAGIC_TOKEN
22	$SCANCOUNT_BASE_MAGIC_TOKEN	22	$SCANCOUNT_BASE_MAGIC_TOKEN $NOW_REBUILDING_MAGIC_TOKEN
23	};	23	};
24		24
25	@ISA = qw();	25	@ISA = qw();
Lines 66-71 Link Here
66	$LAST_EXPIRE_MAGIC_TOKEN = '**LASTEXPIRE';	66	$LAST_EXPIRE_MAGIC_TOKEN = '**LASTEXPIRE';
67	$NTOKENS_MAGIC_TOKEN = '**NTOKENS';	67	$NTOKENS_MAGIC_TOKEN = '**NTOKENS';
68	$SCANCOUNT_BASE_MAGIC_TOKEN = '**SCANBASE';	68	$SCANCOUNT_BASE_MAGIC_TOKEN = '**SCANBASE';
		69	$NOW_REBUILDING_MAGIC_TOKEN = '**NOWREBUILDING';
69		70
70	use constant MAX_SIZE_FOR_SCAN_COUNT_FILE => 5000;	71	use constant MAX_SIZE_FOR_SCAN_COUNT_FILE => 5000;
71		72
Lines 316-321 Link Here
316	\|\| $tok eq $LAST_EXPIRE_MAGIC_TOKEN	317	\|\| $tok eq $LAST_EXPIRE_MAGIC_TOKEN
317	\|\| $tok eq $NTOKENS_MAGIC_TOKEN	318	\|\| $tok eq $NTOKENS_MAGIC_TOKEN
318	\|\| $tok eq $OLDEST_TOKEN_AGE_MAGIC_TOKEN	319	\|\| $tok eq $OLDEST_TOKEN_AGE_MAGIC_TOKEN
		320	\|\| $tok eq $NOW_REBUILDING_MAGIC_TOKEN
319	\|\| $tok eq $SCANCOUNT_BASE_MAGIC_TOKEN);	321	\|\| $tok eq $SCANCOUNT_BASE_MAGIC_TOKEN);
320		322
321	my ($ts, $th, $atime) = $self->tok_get ($tok);	323	my ($ts, $th, $atime) = $self->tok_get ($tok);
Lines 341-348 Link Here
341	}	343	}
342	}	344	}
343		345
344	if ($showdots && (($kept + $deleted) % 1000) == 0) {	346	if ((($kept + $deleted) % 1000) == 0) {
345	print STDERR ".";	347	# update "now expiring" token.
		348	$self->{db_toks}->{$NOW_REBUILDING_MAGIC_TOKEN} = time();
		349	if ($showdots) { print STDERR "."; }
346	}	350	}
347	}	351	}
348		352
Lines 374-379 Link Here
374	$new_toks{$NHAM_MAGIC_TOKEN} = $self->{db_toks}->{$NHAM_MAGIC_TOKEN};	378	$new_toks{$NHAM_MAGIC_TOKEN} = $self->{db_toks}->{$NHAM_MAGIC_TOKEN};
375	$new_toks{$NTOKENS_MAGIC_TOKEN} = $kept + $reprieved;	379	$new_toks{$NTOKENS_MAGIC_TOKEN} = $kept + $reprieved;
376		380
		381	# ensure this is NOT set in the new db
		382	delete $new_toks{$NOW_REBUILDING_MAGIC_TOKEN};
		383
377	# now untie so we can do renames	384	# now untie so we can do renames
378	untie %{$self->{db_toks}};	385	untie %{$self->{db_toks}};
379	untie %new_toks;	386	untie %new_toks;
Lines 391-397 Link Here
391	# ok, once that's done we can re-tie. Call untie_db() first so	398	# ok, once that's done we can re-tie. Call untie_db() first so
392	# we unlock correctly etc. first	399	# we unlock correctly etc. first
393	$self->untie_db();	400	$self->untie_db();
394	$self->tie_db_writable();
395		401
396	my $done = time();	402	my $done = time();
397		403
Lines 427-432 Link Here
427	if (!$ntoks \|\| $ntoks =~ /\D/)	433	if (!$ntoks \|\| $ntoks =~ /\D/)
428	{ $ntoks = $self->{expiry_min_db_size} + 1; }	434	{ $ntoks = $self->{expiry_min_db_size} + 1; }
429	if ($ntoks <= $self->{expiry_min_db_size}) {	435	if ($ntoks <= $self->{expiry_min_db_size}) {
		436	dbg ("bayes: expiry not needed, db too small ($ntoks)");
		437	return 0;
		438	}
		439
		440	# is the database currently being expired, or did an expiry attempt
		441	# fail for some reason in the last 300 seconds?
		442	my $timenow = time();
		443	my $amexping = $self->{db_toks}->{$NOW_REBUILDING_MAGIC_TOKEN};
		444	if (defined ($amexping) && $timenow - $amexping < 300) { # TODO: conf item?
		445	dbg ("bayes: expiry already in progress ($amexping)");
430	return 0;	446	return 0;
431	}	447	}
432		448
Lines 441-446 Link Here
441	return 1;	457	return 1;
442	}	458	}
443		459
		460	dbg ("bayes: expiry not needed, happened recently ($last, now=$now)");
444	0;	461	0;
445	}	462	}
446		463
Lines 583-588 Link Here
583	my $showdots = $opts->{showdots};	600	my $showdots = $opts->{showdots};
584	my $retirepath = $path.".old";	601	my $retirepath = $path.".old";
585		602
		603	# Set this now. Unset it at end of expire_old_tokens_trapped().
		604	# NOTE: we do NOT unset it on error. If something went
		605	# wrong, give the server 300 secs to recover in case it was
		606	# a system-level problem, or caused by load.
		607	$self->{db_toks}->{$NOW_REBUILDING_MAGIC_TOKEN} = $started;
		608
586	# now read the retired journal	609	# now read the retired journal
587	if (!open (JOURNAL, "<$path")) {	610	if (!open (JOURNAL, "<$path")) {
588	warn "bayes: cannot open read $path\n";	611	warn "bayes: cannot open read $path\n";
Lines 613-621 Link Here
613	warn "Bayes journal: gibberish entry found: $_";	636	warn "Bayes journal: gibberish entry found: $_";
614	}	637	}
615		638
616	# if ($showdots && ($count % 1000) == 0) {	639	if (($total_count % 1000) == 0) {
617	# print STDERR ".";	640	# update "now expiring" token.
618	# }	641	$self->{db_toks}->{$NOW_REBUILDING_MAGIC_TOKEN} = time();
		642	# if ($showdots) { print STDERR "."; }
		643	}
619	}	644	}
620	close JOURNAL;	645	close JOURNAL;
621		646
Lines 625-632 Link Here
625	while( my($k,$v) = each %tokens ) {	650	while( my($k,$v) = each %tokens ) {
626	$self->tok_touch_token ($v, $k);	651	$self->tok_touch_token ($v, $k);
627		652
628	if ($showdots && (++$count % 1000) == 0) {	653	if ((++$count % 1000) == 0) {
629	print STDERR ".";	654	# update "now expiring" token.
		655	$self->{db_toks}->{$NOW_REBUILDING_MAGIC_TOKEN} = time();
		656	if ($showdots) { print STDERR "."; }
630	}	657	}
631	}	658	}
632		659