DIS-3017 MARC item prices to recalculate cost savings

code/web/RecordDrivers/RecordDriverFactory.php

@@ -192,4 +192,12 @@ public static function initAndReturnDriver($record, $driver, $path) {
 		// If we got here, something went very wrong:
 		return new AspenError("Problem loading record driver: {$driver}");
 	}
+
+	public static function clearCachedDrivers() {
+		foreach (RecordDriverFactory::$recordDrivers as $recordDriver) {
+			$recordDriver->__destruct();
+			$recordDriver = null;
+		}
+		RecordDriverFactory::$recordDrivers = [];
+	}
 }

code/web/cron/recalculateHistoricCostSavings.php

@@ -10,6 +10,9 @@
 require_once __DIR__ . '/../bootstrap_aspen.php';
 
 set_time_limit(0);
+ini_set('memory_limit', '2G');
+/** @var MemoryWatcher $memoryWatcher */
+global $memoryWatcher;
 
 require_once ROOT_DIR . '/sys/Administration/BackgroundProcess.php';
 $backgroundProcess = null;
@@ -19,7 +22,7 @@
 	$backgroundProcess->id = $backgroundProcessId;
 	if (!$backgroundProcess->find(true)) {
 		$backgroundProcess = null;
-		echo ("Could not find the specified background process\n");
+		if (is_null($backgroundProcess)) { echo ("Could not find the specified background process\n"); }
 		die();
 	}else{
 		if (!$backgroundProcess->isRunning) {
@@ -35,70 +38,116 @@
 require_once ROOT_DIR . '/sys/ReplacementCost.php';
 $replacementCosts = ReplacementCost::getReplacementCostsByFormat();
 
+require_once ROOT_DIR . '/sys/Utils/GroupingUtils.php';
 require_once ROOT_DIR . '/sys/ReadingHistoryEntry.php';
 $readingHistoryEntry = new ReadingHistoryEntry();
 if (!empty($format)) {
 	$readingHistoryEntry->format = $format;
 }
+//Optimize to load by source so we can initialize fewer record drivers
+$readingHistoryEntry->orderBy(['source', 'sourceId']);
 $numEntriesToUpdate = $readingHistoryEntry->count();
 if (!is_null($backgroundProcess)) { $backgroundProcess->addNote("Updating $numEntriesToUpdate reading history entries"); }
+if (is_null($backgroundProcess)) { echo(date('Y M d H:i:s') . " There are $numEntriesToUpdate entries to update " . $memoryWatcher->getCurrentMemoryAllocation()); }
 
 $readingHistoryEntry->find();
 $numUpdated = 0;
+$numProcessed = 0;
 $loggedZeroCostFormats = [];
 
+global $indexingProfiles;
+
 //Recalculate all reading history entries
+$lastSource = '';
+$lastSourceId = '';
+$recordDriver = null;
+global $aspen_db;
 while ($readingHistoryEntry->fetch()) {
+	if ($lastSource != $readingHistoryEntry->source || $lastSourceId != $readingHistoryEntry->sourceId) {
+		if (array_key_exists($lastSource, $indexingProfiles)) {
+			//Clear cache
+			RecordDriverFactory::clearCachedDrivers();
+			$recordDriver = RecordDriverFactory::initRecordDriverById($readingHistoryEntry->source . ':' . $readingHistoryEntry->sourceId);
+		}else{
+			$recordDriver = null;
+		}
+		$lastSource = $readingHistoryEntry->source;
+		$lastSourceId = $readingHistoryEntry->sourceId;
+	}
 	$lowerFormat = strtolower($readingHistoryEntry->format);
-	if (array_key_exists($lowerFormat, $replacementCosts)) {
-		if ($replacementCosts[$lowerFormat] > 0) {
-			//Update the costSavings for the reading history entry and update the total cost savings for the user
-			$readingHistoryEntryToUpdate = new ReadingHistoryEntry();
-			$readingHistoryEntryToUpdate->id = $readingHistoryEntry->id;
-			if ($readingHistoryEntryToUpdate->find(true)) {
-				$readingHistoryEntryToUpdate->costSavings = $replacementCosts[$lowerFormat];
-				$readingHistoryEntryToUpdate->update();
+	//Update the costSavings for the reading history entry and update the total cost savings for the user
+
+	$useFormatMapLookup = true;
+	//We only have replacement costs at the item level for marc records from the ILS so don't bother creating drivers for anything else
+	if (array_key_exists($readingHistoryEntry->source, $indexingProfiles)) {
+		if ($recordDriver != null) {
+			$replacementCost = getReplacementCost($recordDriver, $readingHistoryEntry->format, null, $readingHistoryEntry->barcode);
+			if ($replacementCost > 0) {
+				$aspen_db->exec("UPDATE user_reading_history_work SET costSavings = $replacementCost where id = $readingHistoryEntry->id");
 				$numUpdated++;
+				$useFormatMapLookup = false;
 			}
+			$recordDriver = null;
 		}
-	}else{
-		if (!array_key_exists($lowerFormat, $loggedZeroCostFormats)) {
-			if (!is_null($backgroundProcess)) { $backgroundProcess->addNote("Skipping $readingHistoryEntry->format because no replacement cost was specified."); }
+	}
+	if ($useFormatMapLookup) {
+		$foundCost = false;
+		if (array_key_exists($lowerFormat, $replacementCosts)) {
+			if ($replacementCosts[$lowerFormat] > 0) {
+				//Update the costSavings for the reading history entry and update the total cost savings for the user
+				$replacementCost = $replacementCosts[$lowerFormat];
+				$aspen_db->exec("UPDATE user_reading_history_work SET costSavings = $replacementCost where id = $readingHistoryEntry->id");
+				$numUpdated++;
+				$foundCost = true;
+			}
+		}
+		if (!$foundCost && !array_key_exists($lowerFormat, $loggedZeroCostFormats)) {
+			if (!is_null($backgroundProcess)) {
+				$backgroundProcess->addNote("Skipping $readingHistoryEntry->format because no replacement cost was specified.");
+			}
 			$loggedZeroCostFormats[$lowerFormat] = $lowerFormat;
 		}
 	}
+
+	$numProcessed++;
+	if ($numProcessed % 1000 === 0) {
+		if (is_null($backgroundProcess)) { echo(date('Y M d H:i:s') . " Processed $numProcessed reading history entries "  . $memoryWatcher->getCurrentMemoryAllocation() . "\n"); }
+		if (is_null($backgroundProcess)) { echo(date('Y M d H:i:s') . "  - "  . $memoryWatcher->getCurrentMemoryAllocation() . "\n"); }
+		ob_flush();
+	}
 }
 
 //Now get the total cost savings for users that have checked something out in the format
+if (is_null($backgroundProcess)) { echo(date('Y M d H:i:s') . " Updating total cost savings for users\n"); }
+ob_flush();
 $readingHistoryEntry = new ReadingHistoryEntry();
 if (!empty($format)) {
 	$readingHistoryEntry->format = $format;
 }
+$readingHistoryEntry->whereAdd("costSavings > 0");
 $readingHistoryEntry->selectAdd();
-$readingHistoryEntry->selectAdd("DISTINCT userId as userId");
+$readingHistoryEntry->selectAdd('userId');
+$readingHistoryEntry->selectAdd("sum(costSavings) as costSavings");
+$readingHistoryEntry->groupBy('userId');
+
+$numUsersToUpdate = $readingHistoryEntry->count();
+if (is_null($backgroundProcess)) { echo(date('Y M d H:i:s') . " Updating total cost savings for $numUsersToUpdate users\n"); }
+ob_flush();
 
 $numUsersUpdated = 0;
 $readingHistoryEntry->find();
 while ($readingHistoryEntry->fetch()) {
-	$userToUpdate = new User();
-	$userToUpdate->id = $readingHistoryEntry->userId;
-	if ($userToUpdate->find(true)) {
-		$tmpReadingHistory = new ReadingHistoryEntry();
-		$tmpReadingHistory->userId = $userToUpdate->id;
-		$tmpReadingHistory->selectAdd();
-		$tmpReadingHistory->selectAdd("SUM(costSavings) as costSavings");
-		if ($tmpReadingHistory->costSavings != $userToUpdate->totalCostSavings) {
-			if ($tmpReadingHistory->find(true)) {
-				$userToUpdate->__set('totalCostSavings', $tmpReadingHistory->costSavings);
-			} else {
-				$userToUpdate->__set('totalCostSavings', 0);
-			}
-			$userToUpdate->update();
-		}
-	}
+	$aspen_db->exec("UPDATE user SET totalCostSavings = $readingHistoryEntry->costSavings where id = $readingHistoryEntry->userId");
 	$numUsersUpdated++;
+	if ($numUsersUpdated % 1000 == 0) {
+		if (is_null($backgroundProcess)) { echo(date('Y M d H:i:s') . " Updated $numUsersUpdated users\n"); }
+		ob_flush();
+	}
 }
 
+if (is_null($backgroundProcess)) { echo(date('Y M d H:i:s') . " Done!"); }
+ob_flush();
+
 if (!is_null($backgroundProcess)) {
 	$backgroundProcess->addNote(translate([
 		'text' => 'Updated %1% historic cost savings.',

code/web/release_notes/26.10.00/DIS-3017.yaml

@@ -0,0 +1,12 @@
+Jira: DIS-3017
+Type: Enhancement
+Developer: Mark Noble (MDN)
+ContributingOrg: Grove For Libraries
+Sponsor: Orange County Library System
+GroveRelease: 26.10.00
+CommunityRelease: 26.11.00
+Component: User Account
+Description:
+  - When recalculating historic cost savings, use item prices from MARC records when possible.
+NewSettings:
+NewPermissions:

code/web/sys/Indexing/IlsRecord.php

@@ -72,6 +72,11 @@ static function getIlsRecordForId(string $type, string $identifier) : ?IlsRecord
 			self::$preloadedIlsRecords[$type] = [];
 		}
 		if (!array_key_exists($identifier, self::$preloadedIlsRecords[$type])) {
+			while (count(self::$preloadedIlsRecords[$type]) >= 50) {
+				$firstKey = array_key_first(self::$preloadedIlsRecords[$type]);
+				unset(self::$preloadedIlsRecords[$type][$firstKey]);
+			}
+
 			$ilsRecord = new IlsRecord();
 			$ilsRecord->source = $type;
 			$ilsRecord->ilsId = $identifier;

code/web/sys/MarcLoader.php

@@ -59,8 +59,9 @@ public static function loadMarcRecordByILSId(string $id, string $recordType = 'm
 		}
 		//Make sure not to use too much memory
 		global $memoryWatcher;
-		if (count(MarcLoader::$loadedMarcRecords) > 50) {
-			array_shift(MarcLoader::$loadedMarcRecords);
+		while (count(MarcLoader::$loadedMarcRecords) >= 50) {
+			$firstKey = array_key_first(MarcLoader::$loadedMarcRecords);
+			unset(MarcLoader::$loadedMarcRecords[$firstKey]);
 			$memoryWatcher->logMemory("Removed Cached MARC");
 		}
 		$memoryWatcher->logMemory("Loaded MARC for $id");

code/web/sys/MemoryWatcher.php

@@ -53,6 +53,14 @@ public function writeMemory() {
 		}
 	}
 
+	public function getCurrentMemoryAllocation() {
+		$curMemory = memory_get_usage(false);
+		$memoryInUse = number_format(($curMemory) / 1024 /1024);
+		$curMemory = memory_get_usage(true);
+		$memoryReserved = number_format($curMemory / 1024 /1024);
+		return " $memoryInUse MB in use, $memoryReserved MB reserved";
+	}
+
 	function __destruct() {
 		if ($this->memoryLoggingEnabled) {
 			global $logger;

code/web/sys/User/Checkout.php

@@ -211,53 +211,7 @@ public function update(string $context = '') : int|bool {
 	}
 
 	public function getReplacementCost() : float {
-		require_once ROOT_DIR . '/sys/ReplacementCost.php';
-		$replacementCosts = ReplacementCost::getReplacementCostsByFormat();
-
-		$replacementCostForCheckout = 0;
-		$useDefaultReplacementCost = true;
-		//Check to see if the title has a replacement cost in the record
-		$recordDriver = $this->getRecordDriver();
-		if ($recordDriver instanceof MarcRecordDriver) {
-			$indexingProfile = $recordDriver->getIndexingProfile();
-			if (!empty($indexingProfile->replacementCostSubfield) && $indexingProfile->replacementCostSubfield != ' ' && !empty($indexingProfile->itemRecordNumber) && $indexingProfile->itemRecordNumber != ' '){
-				//We need the full MARC record to get all the data
-				$marcRecord = $recordDriver->getMarcRecord();
-				if ($marcRecord) {
-					$itemFields = $marcRecord->getFields($indexingProfile->itemTag);
-					/** @var File_MARC_Data_Field $itemField */
-					foreach ($itemFields as $itemField) {
-						$recordNumberMatches = false;
-						if (!empty($indexingProfile->itemRecordNumber)) {
-							$itemRecordNumber = $itemField->getSubfield($indexingProfile->itemRecordNumber);
-							$recordNumberMatches = (!empty($itemRecordNumber) && ($itemRecordNumber->getData() == $this->itemId));
-						}
-						$barcodeMatches = false;
-						if (!empty($indexingProfile->barcode)) {
-							$itemBarcode = $itemField->getSubfield($indexingProfile->barcode);
-							$barcodeMatches = (!empty($itemBarcode) && ($itemBarcode->getData() == $this->barcode));
-						}
-						$replacementCost = $itemField->getSubfield($indexingProfile->replacementCostSubfield);
-						if (!empty($replacementCost) && ($recordNumberMatches || $barcodeMatches)) {
-							$replacementCost = $replacementCost->getData();
-							//Remove dollar signs if they are in the field.
-							require_once ROOT_DIR . '/sys/Utils/StringUtils.php';
-							$replacementCost = str_replace(StringUtils::getCurrencySymbol(), '', $replacementCost);
-							if ($replacementCost > 0 && is_numeric($replacementCost)) {
-								$replacementCostForCheckout = $replacementCost;
-								$useDefaultReplacementCost = false;
-							}
-							break;
-						}
-					}
-				}
-				$marcRecord = null;
-			}
-		}
-		$lowerFormat = strtolower($this->format);
-		if ($useDefaultReplacementCost && array_key_exists($lowerFormat, $replacementCosts)) {
-			$replacementCostForCheckout = $replacementCosts[$lowerFormat];
-		}
-		return $replacementCostForCheckout;
+		require_once ROOT_DIR . '/sys/Utils/GroupingUtils.php';
+		return getReplacementCost($this->getRecordDriver(), $this->format, $this->itemId, $this->barcode);
 	}
 }

code/web/sys/Utils/GroupingUtils.php

@@ -282,3 +282,53 @@ function getSortableDate(?string $str) : ?DateTime {
 	$sortableDateCache[$str] = null;
 	return null;
 }
+
+function getReplacementCost(GroupedWorkSubDriver $recordDriver, String $format, ?String $itemId = null, ?String $barcode = null) : float {
+	require_once ROOT_DIR . '/sys/ReplacementCost.php';
+	$replacementCosts = ReplacementCost::getReplacementCostsByFormat();
+
+	$replacementCostForCheckout = 0;
+	$useDefaultReplacementCost = true;
+	//Check to see if the title has a replacement cost in the record
+	if ($recordDriver instanceof MarcRecordDriver) {
+		$indexingProfile = $recordDriver->getIndexingProfile();
+		if (!empty($indexingProfile->replacementCostSubfield) && $indexingProfile->replacementCostSubfield != ' ' && ((!empty($indexingProfile->itemRecordNumber) && $indexingProfile->itemRecordNumber != ' ') || (!empty($indexingProfile->barcode) && $indexingProfile->barcode != ' '))){
+			//We need the full MARC record to get all the data
+			$marcRecord = $recordDriver->getMarcRecord();
+			if ($marcRecord) {
+				$itemFields = $marcRecord->getFields($indexingProfile->itemTag);
+				/** @var File_MARC_Data_Field $itemField */
+				foreach ($itemFields as $itemField) {
+					$recordNumberMatches = false;
+					if (!empty($indexingProfile->itemRecordNumber) && !empty($itemId)) {
+						$itemRecordNumber = $itemField->getSubfield($indexingProfile->itemRecordNumber);
+						$recordNumberMatches = (!empty($itemRecordNumber) && ($itemRecordNumber->getData() == $itemId));
+					}
+					$barcodeMatches = false;
+					if (!empty($indexingProfile->barcode) && !empty($barcode)) {
+						$itemBarcode = $itemField->getSubfield($indexingProfile->barcode);
+						$barcodeMatches = (!empty($itemBarcode) && ($itemBarcode->getData() == $barcode));
+					}
+					$replacementCost = $itemField->getSubfield($indexingProfile->replacementCostSubfield);
+					if (!empty($replacementCost) && ($recordNumberMatches || $barcodeMatches)) {
+						$replacementCost = $replacementCost->getData();
+						//Remove dollar signs if they are in the field.
+						require_once ROOT_DIR . '/sys/Utils/StringUtils.php';
+						$replacementCost = str_replace(StringUtils::getCurrencySymbol(), '', $replacementCost);
+						if ($replacementCost > 0 && is_numeric($replacementCost)) {
+							$replacementCostForCheckout = $replacementCost;
+							$useDefaultReplacementCost = false;
+						}
+						break;
+					}
+				}
+			}
+			$marcRecord = null;
+		}
+	}
+	$lowerFormat = strtolower($format);
+	if ($useDefaultReplacementCost && array_key_exists($lowerFormat, $replacementCosts)) {
+		$replacementCostForCheckout = $replacementCosts[$lowerFormat];
+	}
+	return $replacementCostForCheckout;
+}