FOUR-31977 Multitenant Scheduler Updates

ProcessMaker/Console/Commands/ScheduleMultitenantRun.php

@@ -0,0 +1,111 @@
+<?php
+
+namespace ProcessMaker\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Console\Scheduling\Schedule;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Log;
+use ProcessMaker\Console\Scheduling\FastSchedule;
+use ProcessMaker\Facades\Metrics;
+use ProcessMaker\Multitenancy\Tenant;
+use Throwable;
+
+class ScheduleMultitenantRun extends Command
+{
+    public const LOCK_KEY = 'schedule-multitenant-run';
+
+    public const LOCK_SECONDS = 600; // 10 minutes
+
+    public const DURATION_METRIC = 'schedule_multitenant_run_duration_seconds';
+
+    /**
+     * Histogram buckets in seconds, up to the command lock timeout.
+     */
+    public const DURATION_BUCKETS = [1, 5, 15, 30, 60, 120, 300, 600];
+
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'schedule:tenants-run';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Run the scheduled commands for each tenant';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): int
+    {
+        $lock = Cache::lock(self::LOCK_KEY, self::LOCK_SECONDS);
+
+        if (!$lock->get()) {
+            $message = 'schedule:tenants-run is already running; skipping this invocation.';
+            Log::error($message);
+            $this->error($message);
+
+            return self::FAILURE;
+        }
+
+        $startedAt = microtime(true);
+
+        try {
+            return $this->runSchedules();
+        } catch (Throwable $e) {
+            Log::error('schedule:tenants-run failed: ' . $e->getMessage(), [
+                'exception' => $e,
+            ]);
+
+            throw $e;
+        } finally {
+            $lock->release();
+            Metrics::histogramObserve(
+                self::DURATION_METRIC,
+                'Duration of the multitenant scheduler run in seconds',
+                [],
+                self::DURATION_BUCKETS,
+                microtime(true) - $startedAt
+            );
+        }
+    }
+
+    private function runSchedules(): int
+    {
+        if (config('app.multitenancy') === false) {
+            return $this->call('schedule:run');
+        }
+
+        $schedule = app(Schedule::class);
+        $tenants = Tenant::query()->cursor();
+
+        $ranForTenant = false;
+
+        foreach ($tenants as $tenant) {
+            $ranForTenant = true;
+            $this->info("Running schedule for tenant [{$tenant->id}]");
+
+            try {
+                $tenant->makeCurrent();
+                $this->call('schedule:run');
+            } finally {
+                Tenant::forgetCurrent();
+
+                if ($schedule instanceof FastSchedule) {
+                    $schedule->clearTenantEvents();
+                }
+            }
+        }
+
+        if (!$ranForTenant) {
+            $this->info('No tenants found.');
+        }
+
+        return self::SUCCESS;
+    }
+}

ProcessMaker/Console/Kernel.php

@@ -4,6 +4,7 @@
 
 use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
+use ProcessMaker\Console\Scheduling\FastSchedule;
 
 class Kernel extends ConsoleKernel
 {
@@ -14,6 +15,18 @@ class Kernel extends ConsoleKernel
      */
     protected $commands = [];
 
+    /**
+     * Resolve a console schedule instance that runs Artisan commands in-process.
+     *
+     * @return Schedule
+     */
+    public function resolveConsoleSchedule()
+    {
+        return tap(new FastSchedule($this->scheduleTimezone()), function ($schedule) {
+            $this->schedule($schedule->useCache($this->scheduleCache()));
+        });
+    }
+
     /**
      * Define the application's command schedule.
      *
@@ -24,6 +37,7 @@ protected function schedule(Schedule $schedule)
     {
         $schedule->command('bpmn:timer')
                  ->everyMinute()
+                 ->name('bpmn:timer')
                  ->onOneServer()
                  ->withoutOverlapping(config('app.scheduler.bpmn_timer_overlap_minutes', 5));
 

ProcessMaker/Console/Scheduling/FastCommandEvent.php

@@ -0,0 +1,60 @@
+<?php
+
+namespace ProcessMaker\Console\Scheduling;
+
+use Illuminate\Console\Scheduling\Event;
+use Illuminate\Console\Scheduling\EventMutex;
+use Illuminate\Support\Facades\Artisan;
+use Symfony\Component\Console\Input\StringInput;
+
+class FastCommandEvent extends Event
+{
+    /**
+     * The Artisan command name or signature string.
+     */
+    protected string $artisanCommand;
+
+    /**
+     * Parameters to pass to Artisan::call().
+     *
+     * @var array<string|int, mixed>
+     */
+    protected array $artisanParameters;
+
+    /**
+     * @param  array<string|int, mixed>  $artisanParameters
+     * @param  \DateTimeZone|string|null  $timezone
+     */
+    public function __construct(
+        EventMutex $mutex,
+        string $shellCommand,
+        string $artisanCommand,
+        array $artisanParameters = [],
+        $timezone = null
+    ) {
+        parent::__construct($mutex, $shellCommand, $timezone);
+
+        $this->artisanCommand = $artisanCommand;
+        $this->artisanParameters = $artisanParameters;
+    }
+
+    public function artisanCommandName(): string
+    {
+        return (new StringInput($this->artisanCommand))->getFirstArgument() ?? $this->artisanCommand;
+    }
+
+    /**
+     * Run the command in-process, or shell out when runInBackground is set.
+     *
+     * @param  \Illuminate\Contracts\Container\Container  $container
+     * @return int
+     */
+    protected function execute($container)
+    {
+        if ($this->runInBackground) {
+            return parent::execute($container);
+        }
+
+        return Artisan::call($this->artisanCommand, $this->artisanParameters);
+    }
+}

ProcessMaker/Console/Scheduling/FastSchedule.php

@@ -0,0 +1,155 @@
+<?php
+
+namespace ProcessMaker\Console\Scheduling;
+
+use Illuminate\Console\Application;
+use Illuminate\Console\Scheduling\CacheAware;
+use Illuminate\Console\Scheduling\Schedule;
+use Illuminate\Container\Container;
+use Illuminate\Contracts\Cache\Factory as CacheFactory;
+use LogicException;
+use Symfony\Component\Console\Command\Command as SymfonyCommand;
+
+class FastSchedule extends Schedule
+{
+    /**
+     * The index where the current tenant's scheduled events begin.
+     */
+    private ?int $tenantEventStartIndex = null;
+
+    /**
+     * Begin tracking events registered for the current tenant.
+     *
+     * @return void
+     */
+    public function beginTenantEventRegistration(): void
+    {
+        if ($this->tenantEventStartIndex !== null) {
+            throw new LogicException(
+                'Tenant schedule event registration is already active.'
+            );
+        }
+
+        $this->tenantEventStartIndex = count($this->events);
+        $this->mutexCache = [];
+    }
+
+    /**
+     * Remove events registered for the current tenant.
+     *
+     * @return void
+     */
+    public function clearTenantEvents(): void
+    {
+        if ($this->tenantEventStartIndex === null) {
+            return;
+        }
+
+        $this->events = array_slice($this->events, 0, $this->tenantEventStartIndex);
+        $this->tenantEventStartIndex = null;
+        $this->mutexCache = [];
+    }
+
+    /**
+     * Re-point the event and scheduling mutexes at the given cache factory.
+     *
+     * The mutexes hold a hard reference to the CacheManager resolved when the
+     * schedule was created. When the tenant cache prefix changes, that stale
+     * manager keeps writing mutex keys under the previous prefix, so
+     * onOneServer()/withoutOverlapping() locks leak across tenants. Refreshing
+     * the factory (and clearing the in-process mutex cache) ensures locks use
+     * the currently active, tenant-prefixed cache store.
+     *
+     * @param  CacheFactory|null  $cache
+     * @return void
+     */
+    public function resetCache(?CacheFactory $cache = null): void
+    {
+        $cache ??= Container::getInstance()->make('cache');
+
+        if ($this->eventMutex instanceof CacheAware) {
+            $this->eventMutex->cache = $cache;
+        }
+
+        if ($this->schedulingMutex instanceof CacheAware) {
+            $this->schedulingMutex->cache = $cache;
+        }
+
+        $this->mutexCache = [];
+    }
+
+    /**
+     * Add a new Artisan command event that runs in-process when possible.
+     *
+     * Falls back to a normal shell Event when the command string contains
+     * shell metacharacters (redirects, pipes, etc.).
+     *
+     * @param  SymfonyCommand|string  $command
+     * @param  array  $parameters
+     * @return \Illuminate\Console\Scheduling\Event
+     */
+    public function command($command, array $parameters = [])
+    {
+        $commandDescription = null;
+
+        if ($command instanceof SymfonyCommand) {
+            $command = Container::getInstance()->make(get_class($command));
+            $commandDescription = $command->getDescription();
+            $artisanCommand = $command->getName();
+        } elseif (is_string($command) && class_exists($command)) {
+            $command = Container::getInstance()->make($command);
+            $commandDescription = $command->getDescription();
+            $artisanCommand = $command->getName();
+        } else {
+            $artisanCommand = $command;
+        }
+
+        $artisanSignature = $artisanCommand;
+        if ($parameters !== []) {
+            $artisanSignature .= ' ' . $this->compileParameters($parameters);
+        }
+
+        if ($this->containsShellMetacharacters($artisanSignature)) {
+            $event = parent::command($artisanCommand, $parameters);
+
+            if ($commandDescription !== null) {
+                $event->description($commandDescription);
+            }
+
+            return $event;
+        }
+
+        $shellCommand = Application::formatCommandString($artisanCommand);
+        if ($parameters !== []) {
+            $shellCommand .= ' ' . $this->compileParameters($parameters);
+        }
+
+        $this->events[] = $event = new FastCommandEvent(
+            $this->eventMutex,
+            $shellCommand,
+            $artisanCommand,
+            $parameters,
+            $this->timezone
+        );
+
+        $this->mergePendingAttributes($event);
+
+        if ($commandDescription !== null) {
+            $event->description($commandDescription);
+        }
+
+        if (empty($event->description)) {
+            $event->name($artisanSignature);
+        }
+
+        return $event;
+    }
+
+    /**
+     * Determine if the artisan signature requires a shell (redirects, pipes, etc.).
+     */
+    protected function containsShellMetacharacters(string $command): bool
+    {
+        return (bool) preg_match('/(&&|&|\||>>|>)/', $command);
+    }
+}

ProcessMaker/Facades/Metrics.php

@@ -6,14 +6,14 @@
 use ProcessMaker\Services\MetricsService;
 
 /**
- * @method static \Prometheus\Counter counter(string $name, string $help = null, array $labels = [])
- * @method static \Prometheus\Gauge gauge(string $name, string $help = null, array $labels = [])
- * @method static \Prometheus\Histogram histogram(string $name, string $help = null, array $labels = [], array $buckets = [0.1, 1, 5, 10])
+ * @method static \Prometheus\Counter counter(string $name, string|null $help = null, array $labels = [])
+ * @method static \Prometheus\Gauge gauge(string $name, string|null $help = null, array $labels = [])
+ * @method static \Prometheus\Histogram histogram(string $name, string|null $help = null, array $labels = [], array $buckets = [])
  * @method static void setGauge(string $name, float $value, array $labelValues = [])
  * @method static string renderMetrics()
  * @method static \Prometheus\CollectorRegistry getCollectionRegistry()
- * @method static void counterInc(string $name, string $help = null, array $labels = [])
- * @method static void histogramObserve(string $name, string $help = null, array $labels = [], array $buckets = [0.1, 1, 5, 10], float $executionTime)
+ * @method static void counterInc(string $name, string|null $help = null, array $labels = [])
+ * @method static void histogramObserve(string $name, string|null $help = null, array $labels = [], array $buckets = [], float $executionTime = 0)
  * @method static void clearMetrics()
  */
 class Metrics extends Facade