From c5efea149d40ef10d909b8ff9474210527e70bf4 Mon Sep 17 00:00:00 2001 From: Christoffer Fremling Date: Thu, 13 Aug 2026 23:09:21 -0700 Subject: [PATCH 01/10] fineacquire: wait until acamd reports the goal change applied While ACAM guides, a fine-acquire correction only re-points acamd's goal; the guide loop applies it at its next executed correction. The settle was timed from the send, so with short slicecam exposures the next sample median could be built from pre-move frames and the same correction applied twice (UT 2026-08-06, BD+28d4211: exposed 2.6 arcsec off slit). acamd tracks the request: offsetgoal marks the goal change pending; the guide loop clears it when it executes a correction computed from the changed goal (the pending state is captured before the goal is read, so a correction already in flight can never clear it). The flag is published; do_fineacquire waits for pending false on a status newer than its send before settling. A stop interrupts the wait; timeout FINE_ACQUIRE_MOVE_TIMEOUT (default 60 s, 0 disables) proceeds on the old timer. Also fixes the offset_acam_goal log line, which printed degrees labelled arcsec on the guiding path. Co-Authored-By: Claude Fable 5 --- Config/slicecamd.cfg.in | 7 ++++ acamd/acam_interface.cpp | 20 ++++++++-- acamd/acam_interface.h | 2 + common/message_keys.h | 1 + slicecamd/slicecam_interface.cpp | 68 +++++++++++++++++++++++++++++++- slicecamd/slicecam_interface.h | 2 + 6 files changed, 96 insertions(+), 4 deletions(-) diff --git a/Config/slicecamd.cfg.in b/Config/slicecamd.cfg.in index 3bd6316a..af4bc06b 100644 --- a/Config/slicecamd.cfg.in +++ b/Config/slicecamd.cfg.in @@ -98,6 +98,13 @@ FINE_ACQUIRE_SETTLE_FRAMES=2 # FINE_ACQUIRE_SETTLE_SEC=3.0 +# FINE_ACQUIRE_MOVE_TIMEOUT= +# Max time to wait for acamd to report a requested goal change applied +# before the settle begins. On timeout the settle proceeds anyway. Set to 0 +# to disable the wait. +# +FINE_ACQUIRE_MOVE_TIMEOUT=60.0 + # FINE_ACQUIRE_GAIN= # Proportional gain = {0..1} applied to the commanded offset when the residual # is at or below FINE_ACQUIRE_GAIN_THRESHOLD arcsec. diff --git a/acamd/acam_interface.cpp b/acamd/acam_interface.cpp index f3a0daf1..36dcbdec 100644 --- a/acamd/acam_interface.cpp +++ b/acamd/acam_interface.cpp @@ -1443,6 +1443,7 @@ namespace Acam { const int attempts = this->target.attempts; const std::string filter = this->motion.get_current_filtername(); const std::string cover = this->motion.get_current_coverpos(); + const bool goalshift_pending = this->target.goalshift_pending.load(); // unless forced, only publish if there was a change in any one of these // @@ -1452,7 +1453,8 @@ namespace Acam { nacquired == this->last_status.nacquired && attempts == this->last_status.attempts && filter == this->last_status.filter && - cover == this->last_status.cover ) return; + cover == this->last_status.cover && + goalshift_pending == this->last_status.goalshift_pending ) return; this->last_status.acquire_mode = acquire_mode; this->last_status.is_acquired = is_acquired; @@ -1460,6 +1462,7 @@ namespace Acam { this->last_status.attempts = attempts; this->last_status.filter = filter; this->last_status.cover = cover; + this->last_status.goalshift_pending = goalshift_pending; // assemble the telemetry into a json message // @@ -1473,6 +1476,7 @@ namespace Acam { jmessage_out[Key::Acamd::BACKGROUND] = this->astrometry.get_background(); jmessage_out[Key::Acamd::FILTER] = filter; jmessage_out[Key::Acamd::COVER] = cover; + jmessage_out[Key::Acamd::GOALSHIFT_PENDING] = goalshift_pending; jmessage_out[Key::PUBTIME] = get_time_us(); try { @@ -3609,10 +3613,13 @@ logwrite( function, message.str() ); offset = angular_separation( acam_goal.ra, acam_goal.dec, acam_ra, acam_dec ); } *****/ + const bool goal_change_at_compute = this->goalshift_pending.load(std::memory_order_acquire); + const double goal_ra_used = this->acam_goal.ra; + const double goal_dec_used = this->acam_goal.dec; if ( iface->fpoffsets.solve_offset( acam_ra, acam_dec, - this->acam_goal.ra, this->acam_goal.dec, + goal_ra_used, goal_dec_used, ra_off, dec_off ) == ERROR ) break; - offset = angular_separation( this->acam_goal.ra, this->acam_goal.dec, acam_ra, acam_dec ); + offset = angular_separation( goal_ra_used, goal_dec_used, acam_ra, acam_dec ); message.str(""); message << "[DEBUG] acam_ra=" << acam_ra << " acam_dec=" << acam_dec << " acam_goal.ra=" << acam_goal.ra << " .dec=" << this->acam_goal.dec << " .ang=" << this->acam_goal.angle; @@ -3692,6 +3699,11 @@ logwrite( function, message.str() ); // send offset to TCS here (returns when offset is complete) if ( iface->tcsd.pt_offset( ra_off*3600., dec_off*3600., OFFSETRATE )==ERROR) break; this->allow_large_offset.store(false); // deliberate-offset allowance consumed + if ( goal_change_at_compute && + goal_ra_used == this->acam_goal.ra && + goal_dec_used == this->acam_goal.dec ) { + this->goalshift_pending.store( false, std::memory_order_release ); + } std::this_thread::sleep_for( std::chrono::seconds(1) ); } @@ -5530,6 +5542,8 @@ logwrite( function, message.str() ); // this->fpoffsets.apply_offset( this->target.acam_goal.ra, this->target.dRA, this->target.acam_goal.dec, this->target.dDEC ); + this->target.goalshift_pending.store( true, std::memory_order_release ); + this->publish_status(true); message.str(""); message << this->target.dRA << " " << this->target.dDEC; diff --git a/acamd/acam_interface.h b/acamd/acam_interface.h index 4c2f59fd..06467a0d 100644 --- a/acamd/acam_interface.h +++ b/acamd/acam_interface.h @@ -349,6 +349,7 @@ namespace Acam { std::atomic is_acquired; ///< set if target acquired successfully std::atomic stop_acquisition; ///< set if the acquisition sequence should stop + std::atomic goalshift_pending{false}; ///< a requested goal change awaits execution by the guide loop double tcs_max_offset; double tcs_max_putonslit_offset{300.}; ///< max offset (arcsec) for a deliberate goal offset (put-on-slit etc.) applied while guiding; defaults 300 if ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET absent @@ -530,6 +531,7 @@ namespace Acam { int attempts = 0; std::string filter = ""; std::string cover = ""; + bool goalshift_pending = false; } last_status; public: diff --git a/common/message_keys.h b/common/message_keys.h index 0888bc52..de4e029b 100644 --- a/common/message_keys.h +++ b/common/message_keys.h @@ -106,6 +106,7 @@ namespace Key { inline const std::string ATTEMPTS = "attempts"; inline const std::string SEEING = "seeing"; inline const std::string BACKGROUND = "background"; + inline const std::string GOALSHIFT_PENDING = "goalshift_pending"; ///< a requested goal change awaits execution by the guide loop } namespace Slicecamd { diff --git a/slicecamd/slicecam_interface.cpp b/slicecamd/slicecam_interface.cpp index ec21083e..f1f397ef 100644 --- a/slicecamd/slicecam_interface.cpp +++ b/slicecamd/slicecam_interface.cpp @@ -60,6 +60,10 @@ namespace Slicecam { const bool was_running = this->is_fineacquire_running.load(std::memory_order_acquire); this->is_fineacquire_locked.store(false, std::memory_order_release); this->is_fineacquire_running.store(false, std::memory_order_release); + { + std::lock_guard lock(this->acam_mtx); + this->acam_cv.notify_all(); + } this->publish_status(); logwrite(function, was_running ? "stop requested" : "stopped"); retstring=this->is_fineacquire_running.load(std::memory_order_acquire)?"running":"stopped"; @@ -562,6 +566,9 @@ namespace Slicecam { const double cmd_dra = effective_gain * med_dra; const double cmd_ddec = effective_gain * med_ddec; + const int64_t send_time = get_time_us(); + const bool was_guiding = this->is_acam_guiding.load(); + if ( this->offset_acam_goal( { cmd_dra, cmd_ddec }, true ) != NO_ERROR ) { logwrite( function, "ERROR failed to send offset to ACAM" ); this->is_fineacquire_running.store( false, std::memory_order_release ); @@ -569,6 +576,36 @@ namespace Slicecam { return; } + // while guiding the correction only re-points acamd's goal; wait until + // acamd reports the goal change applied before settling + if ( was_guiding && this->fineacquire_state.move_timeout_sec > 0.0 ) { + std::unique_lock lock(this->acam_mtx); + // completion requires seeing THIS request registered (pending true on a + // status after the send) and then cleared; statuses arrive in publish + // order, so the false cannot predate the request + bool seen_pending = false; + const bool applied = this->acam_cv.wait_for( lock, + std::chrono::duration( this->fineacquire_state.move_timeout_sec ), + [this, send_time, &seen_pending]() { + if ( !this->is_fineacquire_running.load(std::memory_order_acquire) || + !this->should_framegrab_run.load(std::memory_order_acquire) ) return true; + if ( this->last_acam_pubtime.load(std::memory_order_acquire) <= send_time ) return false; + if ( this->acam_goalshift_pending.load(std::memory_order_acquire) ) { + seen_pending = true; + return false; + } + return seen_pending; + }); + if ( !this->should_framegrab_run.load(std::memory_order_acquire) ) return; + if ( !this->is_fineacquire_running.load(std::memory_order_acquire) ) return; + if ( !applied ) { + std::ostringstream w; + w << "WARNING goal change not applied within " + << this->fineacquire_state.move_timeout_sec << " s; proceeding"; + logwrite( function, w.str() ); + } + } + // time-based settle: wait for the TCS to physically finish the move before // sampling resumes. The frame-count settle (settle_frames) is too short in // wall-clock when autoexpose shortens the exposure (e.g. bright targets), so @@ -924,6 +961,10 @@ namespace Slicecam { Common::extract_telemetry_value( jmessage, Key::PUBTIME, pubtime ); this->last_acam_pubtime.store( pubtime, std::memory_order_relaxed ); + bool goalshift_pending=false; + Common::extract_telemetry_value( jmessage, Key::Acamd::GOALSHIFT_PENDING, goalshift_pending ); + this->acam_goalshift_pending.store( goalshift_pending, std::memory_order_relaxed ); + // wake any thread waiting on ACAM state (e.g. fineacquire) std::lock_guard lock(this->acam_mtx); this->acam_cv.notify_all(); @@ -1418,6 +1459,19 @@ namespace Slicecam { applied++; } else + if ( config.param[entry] == "FINE_ACQUIRE_MOVE_TIMEOUT" ) { + try { this->fineacquire_state.move_timeout_sec = std::stod( config.arg[entry] ); } + catch ( const std::exception &e ) { + message.str(""); message << "ERROR invalid FINE_ACQUIRE_MOVE_TIMEOUT " + << config.arg[entry] << ": " << e.what(); + logwrite( function, message.str() ); + return ERROR; + } + message.str(""); message << "SLICECAMD:config:" << config.param[entry] << "=" << config.arg[entry]; + logwrite( function, message.str() ); + applied++; + } + else if ( config.param[entry] == "FINE_ACQUIRE_GAIN" ) { try { this->fineacquire_state.gain = std::stod( config.arg[entry] ); } catch ( const std::exception &e ) { @@ -2101,6 +2155,10 @@ namespace Slicecam { // if ( whattodo == "stop" ) { this->should_framegrab_run.store( false, std::memory_order_release ); // tells framegrab loop to stop + { + std::lock_guard lock(this->acam_mtx); + this->acam_cv.notify_all(); + } if ( this->is_framegrab_running.load(std::memory_order_acquire) ) { // wait for it to stop int wait_ms = std::max( static_cast(3000*(this->camera.andor.begin()->second->camera_info.exptime+1)), 5000 ); // alert user that framegrabbing has stopped @@ -2180,6 +2238,10 @@ namespace Slicecam { // frame will be grabbed. // this->should_framegrab_run.store( false, std::memory_order_release ); + { + std::lock_guard lock(this->acam_mtx); + this->acam_cv.notify_all(); + } if ( this->is_framegrab_running.load(std::memory_order_acquire) ) return; } else @@ -2596,6 +2658,10 @@ namespace Slicecam { // bool is_guiding = this->is_acam_guiding.load(); + // arcsec for the log; the guiding path sends degrees + const double ra_off_arcsec = ra_off * 3600.; + const double dec_off_arcsec = dec_off * 3600.; + // send the offsets now // if ( is_guiding ) { @@ -2632,7 +2698,7 @@ namespace Slicecam { } std::ostringstream message; - message << "requested offsets dRA=" << ra_off << " dDEC=" << dec_off << " arcsec"; + message << "requested offsets dRA=" << ra_off_arcsec << " dDEC=" << dec_off_arcsec << " arcsec"; logwrite(function, message.str()); return NO_ERROR; diff --git a/slicecamd/slicecam_interface.h b/slicecamd/slicecam_interface.h index 90a7fdb1..b3e018c6 100644 --- a/slicecamd/slicecam_interface.h +++ b/slicecamd/slicecam_interface.h @@ -95,6 +95,7 @@ namespace Slicecam { int settle_frames = 0; ///< countdown of frames to discard while telescope settles int settle_count = 2; ///< configured: frames to discard after each move double settle_sec = 3.0; ///< time-based settle (sec) after each move; 0 disables (needed when autoexpose shortens frames so the frame-count settle is too brief in wall-clock) + double move_timeout_sec = 60.0; ///< max wait (sec) for acamd to report the goal change applied; 0 disables the wait int consecutive_centroid_failures = 0; ///< counts consecutive centroid failures // exposure compensation (shared by the reactive trim and, later, autoexpose) double exptime_min = 0.1; ///< clamp: minimum auto-adjusted exposure (sec) @@ -193,6 +194,7 @@ namespace Slicecam { std::atomic is_acam_guiding; ///< is acam guiding? std::atomic last_acam_pubtime{0}; ///< pubtime (us) of latest received acamd status + std::atomic acam_goalshift_pending{false}; ///< acamd has an unexecuted goal change // Latest target (goal) coords published on Topic::TARGETINFO // NAN until a TARGETINFO arrives, so manual runs with no sequencer target log nan. From e3d5a89a87158874fd7c308b3643ec7b41ced607 Mon Sep 17 00:00:00 2001 From: Christoffer Fremling Date: Thu, 20 Aug 2026 00:00:16 -0700 Subject: [PATCH 02/10] fineacquire: single-frame accept on the verification pass After a correction has been applied, one frame already inside the goal is accepted without waiting for the full sample set. A frame can only accept, never reject: larger readings fall through to the existing median logic unchanged. SETTLE_FRAMES config default 2 -> 0; the wall-clock settle after the confirmed goal change covers the move. --- Config/slicecamd.cfg.in | 5 +++-- slicecamd/slicecam_interface.cpp | 33 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/Config/slicecamd.cfg.in b/Config/slicecamd.cfg.in index af4bc06b..c481b4a3 100644 --- a/Config/slicecamd.cfg.in +++ b/Config/slicecamd.cfg.in @@ -86,9 +86,10 @@ FINE_ACQUIRE_MIN_SAMPLES=3 # FINE_ACQUIRE_SETTLE_FRAMES= # frames discarded after each telescope move to allow settling before -# centroid measurements resume. +# centroid measurements resume. The wall-clock settle (FINE_ACQUIRE_SETTLE_SEC, +# applied after acamd confirms the goal change) covers the move itself. # -FINE_ACQUIRE_SETTLE_FRAMES=2 +FINE_ACQUIRE_SETTLE_FRAMES=0 # FINE_ACQUIRE_SETTLE_SEC= # Time-based settle applied after each move, in addition to SETTLE_FRAMES, diff --git a/slicecamd/slicecam_interface.cpp b/slicecamd/slicecam_interface.cpp index f1f397ef..0ee2da88 100644 --- a/slicecamd/slicecam_interface.cpp +++ b/slicecamd/slicecam_interface.cpp @@ -423,6 +423,39 @@ namespace Slicecam { const int max_samples = this->fineacquire_state.max_samples; const int min_samples = this->fineacquire_state.min_samples; + // Verification pass, accept-only early exit: once a correction has been + // applied this run, a single frame already inside the goal is sufficient + // proof. A frame may only ACCEPT -- anything larger falls through to the + // usual median machinery, so a noisy frame can never trigger a + // correction or an extra cycle. + // + if ( this->fineacq_total_dra != 0.0 || this->fineacq_total_ddec != 0.0 ) { + const double r1 = std::hypot( offsets.first, offsets.second ) * 3600.0; + if ( r1 <= this->fineacquire_state.goal_arcsec ) { + std::ostringstream oss; + oss << "fine acquisition converged: single-frame offset r=" << r1 + << " arcsec <= goal=" << this->fineacquire_state.goal_arcsec + << " arcsec (n=" << n << ")"; + logwrite( function, oss.str() ); + + std::ostringstream acqmodel; + acqmodel << "[ACQMODEL] acam2slit dRA=" << this->fineacq_total_dra + << " dDEC=" << this->fineacq_total_ddec << " arcsec" + << " GOALRA=" << this->fineacq_goal_ra + << " GOALDEC=" << this->fineacq_goal_dec + << " CASANGLE=" << this->telem.angle_scope + << " n=" << n + << " cam=" << which; + logwrite( function, acqmodel.str() ); + + this->is_fineacquire_locked.store( true, std::memory_order_release ); + this->is_fineacquire_running.store( false, std::memory_order_release ); + this->fineacquire_state.reset(); + this->publish_status(); + return; + } + } + // wait for the minimum number of samples before evaluating anything // if ( n < min_samples ) return; From d9526bff42eccdebe17a89dc6f6345d3ea25145b Mon Sep 17 00:00:00 2001 From: Christoffer Fremling Date: Thu, 29 Oct 2026 16:09:28 +0200 Subject: [PATCH 03/10] acamd: classify guide, fine-acquire and deliberate-offset corrections A deliberate goal offset (the star-to-target offset after fine acquisition, put-on-slit, the GUI offset) armed a one-shot allowance that any sent correction cleared. A guide correction whose blocking pt_offset was in flight when the offset arrived cleared it as it completed, and the next frame's correction to the new goal was refused at the guiding cap, so the exposure was taken on the offset star. The cap selection also required acquire_mode == TARGET_GUIDE, which is a race of its own. Goal changes now carry a generation, and each correction is classified by the goal it was computed from: guide, fine-acquire, or deliberate offset. Only a deliberate-offset correction may use the larger ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET cap; once sent it has made the change and clears goalshift_pending, unless a newer change arrived meanwhile. A correction computed from an older goal can neither use nor spend a newer goal's allowance. acamd.cfg.in: ACQUIRE_OFFSET_THRESHOLD=1.0, the value the deployed configuration uses. Co-Authored-By: Claude Opus 5.5 --- Config/acamd.cfg.in | 4 +- acamd/acam_interface.cpp | 178 ++++++++++++++++++++++++++------------- acamd/acam_interface.h | 78 ++++++++++++++++- 3 files changed, 196 insertions(+), 64 deletions(-) diff --git a/Config/acamd.cfg.in b/Config/acamd.cfg.in index 3b101dee..8153adf1 100644 --- a/Config/acamd.cfg.in +++ b/Config/acamd.cfg.in @@ -157,10 +157,10 @@ MOTOR_POS="cover 1 1 504 close" # ACQUIRE_TIMEOUT=90 # seconds before ACAM acquisition sequence aborts on failure to acquire (REQUIRED!) ACQUIRE_RETRYS=5 # max number of retrys before acquisition fails (optional, can be left blank to disable) -ACQUIRE_OFFSET_THRESHOLD=0.5 # computed offset below this threshold (in arcsec) defines successful acquisition +ACQUIRE_OFFSET_THRESHOLD=1.0 # computed offset below this threshold (in arcsec) defines successful acquisition ACQUIRE_MIN_REPEAT=2 # minimum number of sequential successful acquires ACQUIRE_TCS_MAX_OFFSET=60 # max offset (arcsec) for an ordinary guiding correction sent to the TCS -ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET=300 # max offset (arcsec) for a deliberate goal offset (put-on-slit, offset-star, end-of-fineacquire, pyGUI 'Offset') applied while guiding +ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET=300 # max offset (arcsec) for the one correction that makes a deliberate goal offset (put-on-slit, offset-star, end-of-fineacquire, pyGUI 'Offset'); guide and fine-acquire corrections keep ACQUIRE_TCS_MAX_OFFSET # SkySimulator options: # SKYSIM_IMAGE_SIZE= where is integer diff --git a/acamd/acam_interface.cpp b/acamd/acam_interface.cpp index 36dcbdec..7e0b47c9 100644 --- a/acamd/acam_interface.cpp +++ b/acamd/acam_interface.cpp @@ -3225,13 +3225,15 @@ logwrite( function, "[DEBUG] acquire here" ); } this->astrometry.solve( last_imagename, this->target.ext_solver_args ); - this->astrometry.get_solution( result, this->target.acam_goal.ra, this->target.acam_goal.dec, this->target.acam_goal.angle, matches, rmsarcsec ); + double solved_ra=NAN, solved_dec=NAN, solved_angle=NAN; + this->astrometry.get_solution( result, solved_ra, solved_dec, solved_angle, matches, rmsarcsec ); + this->target.set_goal( solved_ra, solved_dec, solved_angle ); // under goal_mtx: the guide loop may be reading it if ( result=="NOISY" || result=="GOOD" ) { goodsolution=true; - message.str(""); message << "setting goals from solver: " << this->target.acam_goal.ra << " " - << this->target.acam_goal.dec << " " - << this->target.acam_goal.angle; + message.str(""); message << "setting goals from solver: " << solved_ra << " " + << solved_dec << " " + << solved_angle; logwrite( function, message.str() ); break; } @@ -3311,11 +3313,13 @@ logwrite( function, message.str() ); message.str(""); message << "[DEBUG] input coords_slit.ra="<< this->target.coords_slit.ra << " .dec=" << this->target.coords_slit.dec; logwrite( function, message.str() ); + double goal_ra=NAN, goal_dec=NAN, goal_angle=NAN; this->fpoffsets.compute_offset_last_angle( this->target.get_pointmode(), "ACAM", this->target.coords_slit.ra, this->target.coords_slit.dec, - this->target.acam_goal.ra, this->target.acam_goal.dec, this->target.acam_goal.angle ); + goal_ra, goal_dec, goal_angle ); + this->target.set_goal( goal_ra, goal_dec, goal_angle ); // under goal_mtx: the guide loop may be reading it -message.str(""); message << "[DEBUG] goals: " << this->target.acam_goal.ra << " " << this->target.acam_goal.dec << " " << this->target.acam_goal.angle; +message.str(""); message << "[DEBUG] goals: " << goal_ra << " " << goal_dec << " " << goal_angle; logwrite( function, message.str() ); if ( error == NO_ERROR ) { @@ -3396,7 +3400,10 @@ logwrite( function, message.str() ); this->nacquired = 0; this->attempts = 0; this->sequential_failures = 0; - this->allow_large_offset.store(false); // no stale deliberate-offset allowance + { + std::lock_guard lock(this->goal_mtx); + this->deliberate_gen = 0; // no stale deliberate-offset allowance + } this->is_acquired.store( false, std::memory_order_release ); // Start the timeout clock, initialized as the time now plus the @@ -3470,6 +3477,7 @@ logwrite( function, message.str() ); // an attempt and will keep trying until max attempts. // long error = NO_ERROR; + bool frame_superseded = false; // the goal changed while this frame's correction was computed if ( this->acquire_mode == Acam::TARGET_ACQUIRE || this->acquire_mode == Acam::TARGET_ACQUIRE_HERE || @@ -3525,8 +3533,11 @@ message.str(""); message << "[DEBUG] ACQUIRE_HERE using goals: " << this->acam_g logwrite( function, message.str() ); } ******/ -message.str(""); message << "[DEBUG] using goals: " << this->acam_goal.ra << " " << this->acam_goal.dec << " " << this->acam_goal.angle; +{ +const goal_snapshot_t logged = this->snapshot_goal(); +message.str(""); message << "[DEBUG] using goals: " << logged.ra << " " << logged.dec << " " << logged.angle; logwrite( function, message.str() ); +} message.str(""); message << "[DEBUG] dRA=" << iface->target.dRA << " dDEC=" << iface->target.dDEC; logwrite( function, message.str() ); @@ -3590,7 +3601,8 @@ logwrite( function, message.str() ); // // Offsets calculated as difference between acam goals and the solution from // solve. ACAM goals are the SLIT->ACAM translated coords from the DB and - // are where the ACAM needs to be pointed. Goals never change. + // are where the ACAM needs to be pointed. Goals change only through + // offset_goal and a new acquisition (set_goal); see goal_mtx. // double ra_off, dec_off; // calculated offsets will be in degrees @@ -3613,16 +3625,31 @@ logwrite( function, message.str() ); offset = angular_separation( acam_goal.ra, acam_goal.dec, acam_ra, acam_dec ); } *****/ - const bool goal_change_at_compute = this->goalshift_pending.load(std::memory_order_acquire); - const double goal_ra_used = this->acam_goal.ra; - const double goal_dec_used = this->acam_goal.dec; + // Every correction is classified by the goal it is computed against, read + // together with that goal's generation (PAL-171; see goal_mtx): + // + // - guide correction: the loop has already moved to this goal. Capped at + // ACQUIRE_TCS_MAX_OFFSET and, while guiding, median-filtered. It makes + // no goal change and spends nothing that belongs to one. + // - goal-change correction: offset_goal changed the goal and no correction + // computed against it has been sent. A fine-acquire change + // ('fineguiding') keeps the ACQUIRE_TCS_MAX_OFFSET cap; a deliberate + // change among those still to be made allows + // ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET. Sent at once and alone, never + // through the median filter. Once sent it has made the change(s) and + // clears goalshift_pending, unless a newer change arrived meanwhile. + // + // So a guide correction in flight when a deliberate offset arrives can + // neither use nor spend the deliberate cap. + // + const goal_snapshot_t goal = this->snapshot_goal(); if ( iface->fpoffsets.solve_offset( acam_ra, acam_dec, - goal_ra_used, goal_dec_used, + goal.ra, goal.dec, ra_off, dec_off ) == ERROR ) break; - offset = angular_separation( goal_ra_used, goal_dec_used, acam_ra, acam_dec ); + offset = angular_separation( goal.ra, goal.dec, acam_ra, acam_dec ); message.str(""); message << "[DEBUG] acam_ra=" << acam_ra << " acam_dec=" << acam_dec << " acam_goal.ra=" - << acam_goal.ra << " .dec=" << this->acam_goal.dec << " .ang=" << this->acam_goal.angle; + << goal.ra << " .dec=" << goal.dec << " .ang=" << goal.angle; logwrite( function, message.str() ); message.str(""); message << "[DEBUG] calculated RAoffset=" << ra_off << " DECLoffset=" << dec_off; logwrite( function, message.str() ); @@ -3641,18 +3668,18 @@ logwrite( function, message.str() ); // There is a maximum offset we send to the TCS. This is not a TCS limit // (theirs is very large); it is our safety limit so that a bad solution - // can't move us far off the slit. Ordinary guiding corrections use the - // normal tcs_max_offset (ACQUIRE_TCS_MAX_OFFSET). A deliberate goal offset - // applied while guiding -- via offset_goal, which covers put-on-slit, - // offset-star acquisition, the end-of-fineacquire target offset, and the - // pyGUI 'Offset' button -- is intentionally larger, so for the one - // correction that consumes it we allow up to tcs_max_putonslit_offset - // (ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET). The ACQUIRE path uses tcs_max_offset - // and is unaffected. + // can't move us far off the slit. Guide and fine-acquire corrections use + // tcs_max_offset (ACQUIRE_TCS_MAX_OFFSET). Only the correction that makes + // a deliberate goal offset -- put-on-slit, offset-star acquisition, the + // end-of-fineacquire target offset, the pyGUI 'Offset' button -- may move + // up to tcs_max_putonslit_offset (ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET). // - double maxoffset = this->tcs_max_offset; - if ( this->acquire_mode == Acam::TARGET_GUIDE && this->allow_large_offset.load() ) { - maxoffset = this->tcs_max_putonslit_offset; + const double maxoffset = goal.deliberate ? this->tcs_max_putonslit_offset : this->tcs_max_offset; + + if ( goal.goal_change ) { + message.str(""); message << "NOTICE correction for goal change " << goal.gen << ": offset=" << offset + << " arcsec, cap " << maxoffset << " arcsec"; + logwrite( function, message.str() ); } // Check the requested offset against the applicable max allowed offset @@ -3661,13 +3688,14 @@ logwrite( function, message.str() ); message.str(""); message << "[WARNING] calculated offset " << offset << " not below max " << maxoffset << " and will not be sent to the TCS"; logwrite( function, message.str() ); - this->allow_large_offset.store(false); // deliberate-offset allowance consumed even when rejected + // A refused correction has made no goal change: a deliberate change + // keeps its cap for the next frame. // Match found but failure to send an offset is considered an attempt // so attempts is incremented. // if ( attempts >= this->max_attempts ) { - message.str(""); message << "ERROR: failed to find offset below " << this->tcs_max_offset << " within max number of attempts"; + message.str(""); message << "ERROR: failed to find offset below " << maxoffset << " within max number of attempts"; iface->async.enqueue_and_log( function, message.str() ); error = ERROR; break; @@ -3685,7 +3713,29 @@ logwrite( function, message.str() ); bool should_offset=false; - // While guiding, run the offsets through a median filter over a + // A correction computed against a goal that has changed since is + // neither sent nor filtered: the next frame is computed against the new + // goal. + // + if ( !this->goal_unchanged_since( goal.gen ) ) { + // Measured against a goal that no longer exists: not sent, and the + // frame counts as neither an acquisition nor a failure. + logwrite( function, "NOTICE goal changed while this correction was computed: not sent" ); + attempts = 0; + frame_superseded = true; + break; + } + else + // A goal-change correction is sent at once and alone. It never enters + // the median filter, and the samples there, computed against the old + // goal, are discarded. + // + if ( goal.goal_change ) { + this->reset_offset_params(); + should_offset = true; + } + else + // While guiding, run guide corrections through a median filter over a // specified time period, and send the offsets only at the rate // determined by that period. This returns true and modifies the // values ra_off,dec_off when it's time to offset. @@ -3698,12 +3748,10 @@ logwrite( function, message.str() ); if ( should_offset ) { // send offset to TCS here (returns when offset is complete) if ( iface->tcsd.pt_offset( ra_off*3600., dec_off*3600., OFFSETRATE )==ERROR) break; - this->allow_large_offset.store(false); // deliberate-offset allowance consumed - if ( goal_change_at_compute && - goal_ra_used == this->acam_goal.ra && - goal_dec_used == this->acam_goal.dec ) { - this->goalshift_pending.store( false, std::memory_order_release ); - } + // A goal-change correction has now made its change(s). A guide + // correction makes none: a goal change that arrived while it was in + // flight keeps its own cap and stays pending. + if ( goal.goal_change ) this->goal_change_made( goal.gen ); std::this_thread::sleep_for( std::chrono::seconds(1) ); } @@ -3739,7 +3787,7 @@ logwrite( function, message.str() ); } while ( false ); // the do-loop is executed only once. used to allow breaks. } // end of acquisition sequence - if ( this->nacquired == 0 && this->max_attempts > 0 ) { + if ( !frame_superseded && this->nacquired == 0 && this->max_attempts > 0 ) { logwrite(function,"[DEBUG] nacquired="+std::to_string(nacquired)); if ( ++this->sequential_failures >= this->max_attempts ) { logwrite( function, "ERROR sequential failures "+std::to_string(this->sequential_failures) @@ -5486,8 +5534,9 @@ logwrite( function, message.str() ); /***** Acam::Interface::offset_goal *****************************************/ /** * @brief sets the dRA,dDEC offsets to apply to goal - * @details The offset is not applied here, this only stores the offsets - * in the class. Offsets are applied only while guiding. + * @details This changes the goal only. The acquisition or guide loop + * moves the telescope with its next correction computed against + * the new goal (see goal_mtx in Target). * @param[in] args input string expected " " * @param[out] retstring return string * @return ERROR | NO_ERROR @@ -5509,7 +5558,7 @@ logwrite( function, message.str() ); retstring = ACAMD_OFFSETGOAL; retstring.append( " [ [ fineguiding ] ]\n" ); retstring.append( " Apply offsets to the ACAM goal coordinates.\n" ); - retstring.append( " These offsets are applied only while guiding. If omitted,\n" ); + retstring.append( " The acquisition or guide loop then moves to the new goal. If omitted,\n" ); retstring.append( " the current offsets are returned. Units are in degrees.\n" ); retstring.append( " 'fineguiding' marks a slicecam fine-guiding correction (keeps the 60\" cap).\n" ); return HELP; @@ -5524,29 +5573,48 @@ logwrite( function, message.str() ); retstring="invalid_argument"; return ERROR; } - this->target.dRA = dRA; - this->target.dDEC = dDEC; - - // optional fineguiding flag: marks a slicecam fine-guiding correction, which + // optional fineguiding flag: marks a slicecam fine-acquire correction, which // must stay within the ordinary 60" guiding cap. Its absence means a // deliberate goal offset (put-on-slit, offset-star, end-of-fineacquire, // pyGUI 'Offset') that may legitimately be larger. std::string flag; const bool is_fineguiding = (iss >> flag && flag == "fineguiding"); - // Apply any dRA, dDEC goal offsets from the "put on slit" action to - // acam_ra_goal, acam_dec_goal. These dRA,dDEC offsets can come from - // either the ACAM or slicecam GUIs and are stored in the Target class. - // - // The supplied acam_ra_goal,acam_dec_goal are modified by dRA,dDEC + // Apply the dRA,dDEC offsets to the goal and record the change with its + // kind, in one step under goal_mtx: the guide loop never reads half a + // change, and two offsets arriving on different connections are both kept. + // Only a correction the guide loop computes against this goal (or a later + // one) can make the change; for a deliberate offset that correction alone + // may move up to ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET, whatever mode the loop is + // in. A correction already in flight can neither use nor spend it (PAL-171). // - this->fpoffsets.apply_offset( this->target.acam_goal.ra, this->target.dRA, - this->target.acam_goal.dec, this->target.dDEC ); - this->target.goalshift_pending.store( true, std::memory_order_release ); + unsigned long long gen=0; + { + std::lock_guard lock( this->target.goal_mtx ); + double goal_ra = this->target.acam_goal.ra; + double goal_dec = this->target.acam_goal.dec; + if ( this->fpoffsets.apply_offset( goal_ra, dRA, goal_dec, dDEC ) == ERROR ) { + logwrite( function, "ERROR applying the offset to the goal: goal unchanged" ); + retstring="goal_unchanged"; + return ERROR; + } + this->target.acam_goal.ra = goal_ra; + this->target.acam_goal.dec = goal_dec; + this->target.dRA = dRA; + this->target.dDEC = dDEC; + gen = ++this->target.goal_gen; + if ( !is_fineguiding && ( dRA != 0.0 || dDEC != 0.0 ) ) { + this->target.deliberate_gen = gen; // a zero offset changes nothing, so it raises no cap + } + this->target.goalshift_pending.store( true, std::memory_order_release ); + } this->publish_status(true); + message.str(""); message << "goal change " << gen << " (" << ( is_fineguiding ? "fine-acquire" : "deliberate offset" ) + << "): dRA=" << dRA*3600. << " dDEC=" << dDEC*3600. << " arcsec"; + logwrite( function, message.str() ); - message.str(""); message << this->target.dRA << " " << this->target.dDEC; + message.str(""); message << dRA << " " << dDEC; retstring = message.str(); // Applying an offset to the goal while guiding must never drop out of @@ -5554,14 +5622,8 @@ logwrite( function, message.str() ); // TARGET_GUIDE and reset the offset filter so the new goal takes effect // quickly. This covers GUI "put on slit" and sequencer target offsets. // - // This is a deliberate offset (put-on-slit, offset-star, end-of-fineacquire, - // or the pyGUI 'Offset' button) and may exceed the normal guiding cap, so - // allow the next correction up to ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET. The - // allowance is one-shot: do_acquire consumes it when the offset is sent. - // if ( this->target.acquire_mode == Acam::TARGET_GUIDE ) { this->target.reset_offset_params(); - if ( !is_fineguiding ) this->target.allow_large_offset.store(true); } this->publish_status(); diff --git a/acamd/acam_interface.h b/acamd/acam_interface.h index 06467a0d..4648bffa 100644 --- a/acamd/acam_interface.h +++ b/acamd/acam_interface.h @@ -352,7 +352,7 @@ namespace Acam { std::atomic goalshift_pending{false}; ///< a requested goal change awaits execution by the guide loop double tcs_max_offset; - double tcs_max_putonslit_offset{300.}; ///< max offset (arcsec) for a deliberate goal offset (put-on-slit etc.) applied while guiding; defaults 300 if ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET absent + double tcs_max_putonslit_offset{300.}; ///< max offset (arcsec) for the correction that makes a deliberate goal offset (put-on-slit etc.); defaults 300 if ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET absent double offset_cal_offset, offset_cal_raoff, offset_cal_decoff; @@ -387,7 +387,14 @@ namespace Acam { this->time_offs.clear(); this->tcs_offset_period = val; } - inline void reset_offset_params() { this->reset_offset_params(this->tcs_offset_period); } + /// clear the filter only: never rewrite tcs_offset_period, which a concurrent + /// offsetperiod command may be setting + inline void reset_offset_params() { + std::lock_guard lock(this->offset_params_mtx); + this->ra_offs.clear(); + this->dec_offs.clear(); + this->time_offs.clear(); + } inline double get_timeout() { return this->timeout; } @@ -484,9 +491,72 @@ namespace Acam { double ra; double dec; double angle; - } acam_goal; + } acam_goal; ///< guarded by goal_mtx (snapshot_goal, set_goal, offset_goal) - std::atomic allow_large_offset{false}; ///< one-shot: allow the next guiding correction up to tcs_max_putonslit_offset + // Goal changes and the corrections that make them (PAL-171). + // + // offset_goal changes the goal; the guide loop moves the telescope. Every + // correction is classified by the goal it was computed against: + // guide correction -- a goal the loop has already moved to; + // goal-change correction -- a goal changed since: a fine-acquire + // correction ('fineguiding') or a deliberate + // offset (untagged: target offset, offset star, + // put-on-slit, pyGUI 'Offset'). + // goal_gen counts goal changes (offset_goal) and replacements (set_goal, + // which counts as made). goal_gen_applied is the newest change that a SENT + // correction was computed against. deliberate_gen is the newest + // deliberate change (0 = none); it is still to be made while + // deliberate_gen > goal_gen_applied, and only a correction computed against + // it (or a later goal) can make it -- so a correction already in flight can + // neither use nor spend its cap. All guarded by goal_mtx. + // + std::mutex goal_mtx; + unsigned long long goal_gen{0}; + unsigned long long goal_gen_applied{0}; + unsigned long long deliberate_gen{0}; + + struct goal_snapshot_t { + double ra, dec, angle; + unsigned long long gen; ///< goal generation the correction is computed against + bool goal_change; ///< includes a goal change not yet made + bool deliberate; ///< includes a deliberate goal change not yet made + }; + + /// the goal, its generation and its classification, read together + inline goal_snapshot_t snapshot_goal() { + std::lock_guard lock(this->goal_mtx); + return { this->acam_goal.ra, this->acam_goal.dec, this->acam_goal.angle, this->goal_gen, + this->goal_gen > this->goal_gen_applied, + this->deliberate_gen > this->goal_gen_applied }; + } + + /// true if no goal change has arrived since the snapshot of generation gen + inline bool goal_unchanged_since( unsigned long long gen ) { + std::lock_guard lock(this->goal_mtx); + return this->goal_gen == gen; + } + + /// a correction computed against generation gen has been sent: the goal + /// changes up to gen are made. goalshift_pending stays set if a newer one arrived. + inline void goal_change_made( unsigned long long gen ) { + std::lock_guard lock(this->goal_mtx); + if ( gen > this->goal_gen_applied ) this->goal_gen_applied = gen; + this->goalshift_pending.store( this->goal_gen > this->goal_gen_applied, std::memory_order_release ); + } + + /// a new acquisition replaces the goal (from the target coordinates or a + /// solve): every earlier change is void and nothing is pending. The + /// generation moves on, so a correction computed against the replaced goal + /// is dropped; the new goal counts as made (the acquisition moves to it). + inline void set_goal( const double ra, const double dec, const double angle ) { + std::lock_guard lock(this->goal_mtx); + this->acam_goal.ra = ra; + this->acam_goal.dec = dec; + this->acam_goal.angle = angle; + this->goal_gen_applied = ++this->goal_gen; + this->deliberate_gen = 0; + this->goalshift_pending.store( false, std::memory_order_release ); + } Target() : iface(nullptr), timeout(10), max_attempts(-1), min_repeat(1), is_acquired(false), From a9a6f0a22b5391aa3d965848ac96a47415103780 Mon Sep 17 00:00:00 2001 From: Christoffer Fremling Date: Thu, 29 Oct 2026 16:08:58 +0200 Subject: [PATCH 04/10] slicecamd: fine-acquire correction budget A target that kept landing just outside the goal was corrected again and again until the sequencer's acquisition timeout. Two slicecamd.cfg keys: FINE_ACQUIRE_MAX_ADJUSTMENTS= (default 3, 0 = previous behaviour): at most corrections are commanded in one run, and the -th decides. It is sent only if the offset it removes is within FINE_ACQUIRE_RELAXED_FACTOR x goal, and the run converges once that move has executed and settled; otherwise the run fails at once without sending it. A correction acamd does not confirm within FINE_ACQUIRE_MOVE_TIMEOUT also fails a budgeted run. Nothing is sampled or evaluated once no correction can follow. FINE_ACQUIRE_RELAXED_FACTOR= (default 2.0, at least 1). Also: a single frame inside the goal after a correction converges the run; FINE_ACQUIRE_SETTLE_SEC=1 (was 3.0) now that the settle starts when the move has executed; every lock writes the same "fine acquisition converged" line, with how it locked appended. Co-Authored-By: Claude Opus 5.5 --- Config/slicecamd.cfg.in | 37 +++++++-- slicecamd/slicecam_interface.cpp | 137 ++++++++++++++++++++++++++++++- slicecamd/slicecam_interface.h | 3 + 3 files changed, 166 insertions(+), 11 deletions(-) diff --git a/Config/slicecamd.cfg.in b/Config/slicecamd.cfg.in index c481b4a3..6bd4d003 100644 --- a/Config/slicecamd.cfg.in +++ b/Config/slicecamd.cfg.in @@ -92,17 +92,38 @@ FINE_ACQUIRE_MIN_SAMPLES=3 FINE_ACQUIRE_SETTLE_FRAMES=0 # FINE_ACQUIRE_SETTLE_SEC= -# Time-based settle applied after each move, in addition to SETTLE_FRAMES, -# to let the TCS physically finish the move before sampling resumes. This matters -# when autoexpose shortens the exposure (e.g. bright targets): the frame-count -# settle then elapses too quickly in wall-clock. Set to 0 to disable. -# -FINE_ACQUIRE_SETTLE_SEC=3.0 +# Pause after each correction, before the next exposure starts. The frame +# loop already waits for acamd to report the move made, which is only after the +# TCS has answered it (#479), so no frame can overlap the move. acamd usually +# reports 1 s after the answer, but sooner if anything requests its status in +# that second; this pause guarantees the settle either way. Set to 0 to disable. +# +FINE_ACQUIRE_SETTLE_SEC=1 + +# FINE_ACQUIRE_MAX_ADJUSTMENTS= +# Correction budget. At most corrections are commanded in one run, and the +# -th decides. It is sent only if the offset it removes is within +# FINE_ACQUIRE_RELAXED_FACTOR x goal; the run then converges once that move has +# executed and settled ("converged (FINAL CORRECTION)"). If the offset is +# larger, the run fails at once without sending it ("fine acquisition FAILED: +# correction of would be the last ..."). A correction acamd does not +# confirm within FINE_ACQUIRE_MOVE_TIMEOUT also fails a budgeted run. The +# sequencer reports a failure as a failed acquisition. Nothing is sampled or +# evaluated once no correction can follow. Needs FINE_ACQUIRE_MOVE_TIMEOUT > 0. +# 0 disables (correct until the goal or the sequencer timeout). +FINE_ACQUIRE_MAX_ADJUSTMENTS=3 + +# FINE_ACQUIRE_RELAXED_FACTOR= +# Largest offset, as a multiple of the 0.4" fine-acquire goal, that the budget's +# last correction may remove: factor 2.0 sends it, and then converges, only if +# that offset is <= 0.8". Must be >= 1 (1 means the last correction is never sent). +FINE_ACQUIRE_RELAXED_FACTOR=2.0 # FINE_ACQUIRE_MOVE_TIMEOUT= # Max time to wait for acamd to report a requested goal change applied -# before the settle begins. On timeout the settle proceeds anyway. Set to 0 -# to disable the wait. +# before the settle begins. On timeout the settle proceeds anyway, except under +# a correction budget, where the run fails. Set to 0 to disable the wait (not +# allowed with FINE_ACQUIRE_MAX_ADJUSTMENTS > 0). # FINE_ACQUIRE_MOVE_TIMEOUT=60.0 diff --git a/slicecamd/slicecam_interface.cpp b/slicecamd/slicecam_interface.cpp index 0ee2da88..edc2420f 100644 --- a/slicecamd/slicecam_interface.cpp +++ b/slicecamd/slicecam_interface.cpp @@ -169,6 +169,7 @@ namespace Slicecam { this->fineacquire_state.reset(); this->fineacq_total_dra = 0.0; // reset per-run ACAM->slit residual accumulators this->fineacq_total_ddec = 0.0; + this->fineacq_adjustments = 0; // relaxed-accept counter, per run // snapshot this run's goal (DB target) coords from the TARGETINFO published message; NAN if no target // has been published (manual runs log nan). Frozen for the run via the is_fineacquire_running handoff. @@ -432,10 +433,14 @@ namespace Slicecam { if ( this->fineacq_total_dra != 0.0 || this->fineacq_total_ddec != 0.0 ) { const double r1 = std::hypot( offsets.first, offsets.second ) * 3600.0; if ( r1 <= this->fineacquire_state.goal_arcsec ) { + // the same converged line as every lock (log readers key on it) std::ostringstream oss; - oss << "fine acquisition converged: single-frame offset r=" << r1 - << " arcsec <= goal=" << this->fineacquire_state.goal_arcsec - << " arcsec (n=" << n << ")"; + oss << "fine acquisition converged: offset dRA=" << offsets.first * 3600.0 + << " dDEC=" << offsets.second * 3600.0 + << " arcsec (r=" << r1 + << " arcsec, n=1 scatter=(0,0) arcsec)" + << " goal=" << this->fineacquire_state.goal_arcsec << " arcsec" + << "; accepted on a single frame after a correction"; logwrite( function, oss.str() ); std::ostringstream acqmodel; @@ -578,6 +583,30 @@ namespace Slicecam { } } + // THE BUDGET'S LAST CORRECTION (FINE_ACQUIRE_MAX_ADJUSTMENTS): the run can + // lock after it only if the offset it would remove is within + // FINE_ACQUIRE_RELAXED_FACTOR x goal, and that offset is known now. When it + // is larger the run fails here, without sending a correction that no lock + // can follow (the sequencer reports a failed acquisition; the robot retries). + // + if ( this->fineacquire_state.max_adjustments > 0 + && this->fineacq_adjustments + 1 >= this->fineacquire_state.max_adjustments + && !( offset_arcsec <= this->fineacquire_state.relaxed_factor + * this->fineacquire_state.goal_arcsec ) ) { + std::ostringstream oss; + oss << "fine acquisition FAILED: correction " << this->fineacq_adjustments + 1 + << " of " << this->fineacquire_state.max_adjustments << " would be the last and r=" + << offset_arcsec << " arcsec > relaxed goal=" + << this->fineacquire_state.relaxed_factor * this->fineacquire_state.goal_arcsec + << " arcsec, so no lock can follow; not sent"; + logwrite( function, oss.str() ); + this->is_fineacquire_locked.store( false, std::memory_order_release ); + this->is_fineacquire_running.store( false, std::memory_order_release ); + this->fineacquire_state.reset(); + this->publish_status(); + return; + } + // select gain: use gain_large when offset is well above the goal threshold // const double effective_gain = ( offset_arcsec > this->fineacquire_state.gain_threshold_arcsec ) @@ -608,6 +637,7 @@ namespace Slicecam { this->publish_status(); return; } + this->fineacq_adjustments++; // one more commanded correction this run (relaxed accept) // while guiding the correction only re-points acamd's goal; wait until // acamd reports the goal change applied before settling @@ -633,6 +663,20 @@ namespace Slicecam { if ( !this->is_fineacquire_running.load(std::memory_order_acquire) ) return; if ( !applied ) { std::ostringstream w; + if ( this->fineacquire_state.max_adjustments > 0 ) { + // A budgeted run decides on the corrections it has commanded, so it + // cannot go on from one that never executed: later frames would + // still show the old position, and the shift may land mid-exposure. + w << "fine acquisition FAILED: correction " << this->fineacq_adjustments << " of " + << this->fineacquire_state.max_adjustments << " was not confirmed applied by acamd within " + << this->fineacquire_state.move_timeout_sec << " s"; + logwrite( function, w.str() ); + this->is_fineacquire_locked.store( false, std::memory_order_release ); + this->is_fineacquire_running.store( false, std::memory_order_release ); + this->fineacquire_state.reset(); + this->publish_status(); + return; + } w << "WARNING goal change not applied within " << this->fineacquire_state.move_timeout_sec << " s; proceeding"; logwrite( function, w.str() ); @@ -646,11 +690,55 @@ namespace Slicecam { if ( this->fineacquire_state.settle_sec > 0.0 ) { std::this_thread::sleep_for( std::chrono::duration( this->fineacquire_state.settle_sec ) ); } + // a stop that arrived during the settle ends this run here: nothing below + // (totals, the budget's decision, a published lock) may act for it + if ( !this->should_framegrab_run.load(std::memory_order_acquire) || + !this->is_fineacquire_running.load(std::memory_order_acquire) ) return; // accumulate the applied correction (arcsec). Summed over the run this is the // total ACAM->slit residual that acam-acquire left behind (the [ACQMODEL] line). this->fineacq_total_dra += cmd_dra * 3600.0; this->fineacq_total_ddec += cmd_ddec * 3600.0; + // The budget's last correction decides. Once FINE_ACQUIRE_MAX_ADJUSTMENTS + // corrections have been commanded, nothing further is sampled or evaluated. + // This last one was only sent because the offset it removes is within + // FINE_ACQUIRE_RELAXED_FACTOR x goal (checked before the send), and it has + // executed and settled, so the run converges now. 0 disables the budget + // (correct until the goal or the sequencer timeout). + // + if ( this->fineacquire_state.max_adjustments > 0 + && this->fineacq_adjustments >= this->fineacquire_state.max_adjustments ) { + std::ostringstream oss; + // the same converged line as every lock (log readers key on it); the + // offset is the one this last correction was sent for, and the goal the + // relaxed one it was accepted against + oss << "fine acquisition converged: offset dRA=" << med_dra * 3600.0 + << " dDEC=" << med_ddec * 3600.0 + << " arcsec (r=" << offset_arcsec + << " arcsec, n=" << n + << " scatter=(" << sig_dra << "," << sig_ddec << ") arcsec)" + << " goal=" << this->fineacquire_state.relaxed_factor * this->fineacquire_state.goal_arcsec + << " arcsec; accepted on the final correction (" << this->fineacq_adjustments + << " of " << this->fineacquire_state.max_adjustments << ") without a verification frame"; + logwrite( function, oss.str() ); + + std::ostringstream acqmodel; + acqmodel << "[ACQMODEL] acam2slit dRA=" << this->fineacq_total_dra + << " dDEC=" << this->fineacq_total_ddec << " arcsec" + << " GOALRA=" << this->fineacq_goal_ra + << " GOALDEC=" << this->fineacq_goal_dec + << " CASANGLE=" << this->telem.angle_scope + << " n=" << n + << " cam=" << which; + logwrite( function, acqmodel.str() ); + + this->is_fineacquire_locked.store( true, std::memory_order_release ); + this->is_fineacquire_running.store( false, std::memory_order_release ); + this->fineacquire_state.reset(); + this->publish_status(); + return; + } + // reset samples and discard settle_count frames for telescope settling this->fineacquire_state.reset(); this->fineacquire_state.settle_frames = this->fineacquire_state.settle_count; @@ -1492,6 +1580,40 @@ namespace Slicecam { applied++; } else + if ( config.param[entry] == "FINE_ACQUIRE_MAX_ADJUSTMENTS" ) { + try { this->fineacquire_state.max_adjustments = std::stoi( config.arg[entry] ); } + catch ( const std::exception &e ) { + message.str(""); message << "ERROR invalid FINE_ACQUIRE_MAX_ADJUSTMENTS " + << config.arg[entry] << ": " << e.what(); + logwrite( function, message.str() ); + return ERROR; + } + if ( this->fineacquire_state.max_adjustments < 0 ) { + logwrite( function, "ERROR FINE_ACQUIRE_MAX_ADJUSTMENTS must be >= 0 (0 disables the budget)" ); + return ERROR; + } + message.str(""); message << "SLICECAMD:config:" << config.param[entry] << "=" << config.arg[entry]; + logwrite( function, message.str() ); + applied++; + } + else + if ( config.param[entry] == "FINE_ACQUIRE_RELAXED_FACTOR" ) { + try { this->fineacquire_state.relaxed_factor = std::stod( config.arg[entry] ); } + catch ( const std::exception &e ) { + message.str(""); message << "ERROR invalid FINE_ACQUIRE_RELAXED_FACTOR " + << config.arg[entry] << ": " << e.what(); + logwrite( function, message.str() ); + return ERROR; + } + if ( !( this->fineacquire_state.relaxed_factor >= 1.0 ) ) { // also refuses NaN + logwrite( function, "ERROR FINE_ACQUIRE_RELAXED_FACTOR must be a number >= 1" ); + return ERROR; + } + message.str(""); message << "SLICECAMD:config:" << config.param[entry] << "=" << config.arg[entry]; + logwrite( function, message.str() ); + applied++; + } + else if ( config.param[entry] == "FINE_ACQUIRE_MOVE_TIMEOUT" ) { try { this->fineacquire_state.move_timeout_sec = std::stod( config.arg[entry] ); } catch ( const std::exception &e ) { @@ -1721,6 +1843,15 @@ namespace Slicecam { } } + // a correction budget decides on the moves it commands, so it needs the + // confirmation that each one executed (the acamd move wait) + // + if ( this->fineacquire_state.max_adjustments > 0 && + !( this->fineacquire_state.move_timeout_sec > 0.0 ) ) { + logwrite( function, "ERROR FINE_ACQUIRE_MAX_ADJUSTMENTS > 0 needs FINE_ACQUIRE_MOVE_TIMEOUT > 0" ); + return ERROR; + } + // min_snr, when set, must be at least 2 // NAN (unset) disables SNR override // diff --git a/slicecamd/slicecam_interface.h b/slicecamd/slicecam_interface.h index b3e018c6..3a5eb9eb 100644 --- a/slicecamd/slicecam_interface.h +++ b/slicecamd/slicecam_interface.h @@ -95,6 +95,8 @@ namespace Slicecam { int settle_frames = 0; ///< countdown of frames to discard while telescope settles int settle_count = 2; ///< configured: frames to discard after each move double settle_sec = 3.0; ///< time-based settle (sec) after each move; 0 disables (needed when autoexpose shortens frames so the frame-count settle is too brief in wall-clock) + int max_adjustments = 0; ///< correction budget: at most this many corrections per run; the last is sent only if the offset it removes is <= relaxed_factor x goal_arcsec, and the run converges once it has executed, else fails; 0 disables + double relaxed_factor = 2.0; ///< correction budget: multiple of goal_arcsec the last correction may remove double move_timeout_sec = 60.0; ///< max wait (sec) for acamd to report the goal change applied; 0 disables the wait int consecutive_centroid_failures = 0; ///< counts consecutive centroid failures // exposure compensation (shared by the reactive trim and, later, autoexpose) @@ -150,6 +152,7 @@ namespace Slicecam { // the ACAM->slit pointing offset vs geometry over time. Touched only from the // single framegrab thread, so plain doubles are sufficient. double fineacq_total_dra = 0.0; ///< sum of applied dRA corrections this run [arcsec] + int fineacq_adjustments = 0; ///< corrections commanded this run (correction budget counter; survives FineAcqState::reset()) double fineacq_total_ddec = 0.0; ///< sum of applied dDEC corrections this run [arcsec] double fineacq_goal_ra = NAN; ///< per-run snapshot of goal RA [deg], taken at fineacquire start, logged at lock double fineacq_goal_dec = NAN; ///< per-run snapshot of goal DEC [deg], taken at fineacquire start, logged at lock From 1c470fb51a0bd946667ced07043add0f956cd40d Mon Sep 17 00:00:00 2001 From: Christoffer Fremling Date: Tue, 25 Aug 2026 23:36:16 -0700 Subject: [PATCH 05/10] auto-ontarget: the TCS itself may satisfy the ontarget wait (opt-in) ?ONTARGET was invisible through tcsd because send_command translates every non-whitelisted reply as a status code: the flag IS the reply, so 0 collided with TCS_SUCCESS ('success') and 1 fell through to 'tcs_undefined' ERROR. Whitelist it for verbatim passthrough. sequencerd: config TCS_AUTO_ONTARGET (default no) spawns a watcher for exactly the lifetime of move_to_target's TCSOP wait. It calls ontarget() -- the operator's own path -- when, on two consecutive 1 Hz reads: ?ONTARGET==1 AND motion==tracking AND |telescope-target| < TCS_AUTO_ONTARGET_SEP arcsec. All three are required: shadow data (UT 2026-08-25, 64 slews) shows the flag is position-agnostic, the separation cannot see sub-threshold hops, and motion says nothing about settle. Operator ack trailed the flag by median +9.7 s / p90 +18 s and never once preceded it. The operator's press is unchanged; whichever arrives first wins. --- Config/sequencerd.cfg.in | 2 + sequencerd/sequence.cpp | 96 +++++++++++++++++++++++++++++++++ sequencerd/sequence.h | 8 +++ sequencerd/sequencer_server.cpp | 25 +++++++++ tcsd/tcs_interface.cpp | 3 ++ 5 files changed, 134 insertions(+) diff --git a/Config/sequencerd.cfg.in b/Config/sequencerd.cfg.in index e665eeb6..d8db2de4 100644 --- a/Config/sequencerd.cfg.in +++ b/Config/sequencerd.cfg.in @@ -150,6 +150,8 @@ TCS_OFFSET_RATE_RA=45 # TCS offset rate for RA ("MRATE") in arcse TCS_OFFSET_RATE_DEC=45 # TCS offset rate for DEC ("MRATE") in arcsec per second TCS_SETTLE_TIMEOUT=20 # seconds to wait for telescope to settle (for "TRACKING_STABLY") TCS_SETTLE_STABLE=1.5 # time in fractional seconds (to 0.1) that TCS must report TRACKING before it is really stable +TCS_AUTO_ONTARGET=no # yes: TCS ?ONTARGET=1 + tracking + sep < TCS_AUTO_ONTARGET_SEP (2 consecutive 1 Hz reads) satisfies the ontarget wait; operator press always works +TCS_AUTO_ONTARGET_SEP=5.0 # arcsec: max |telescope-target| for auto-ontarget TCS_DOMEAZI_READY=10 # max degrees azimuth that dome and telescope can differ before ready to observe TCS_PREAUTH_TIME=10 # seconds before end of exposure to notify TCS of next target's coords (0 to disable) diff --git a/sequencerd/sequence.cpp b/sequencerd/sequence.cpp index 338b6077..864048f7 100644 --- a/sequencerd/sequence.cpp +++ b/sequencerd/sequence.cpp @@ -2149,6 +2149,88 @@ namespace Sequencer { /***** Sequencer::Sequence::camera_shutdown *********************************/ + /***** Sequencer::Sequence::dothread_auto_ontarget **************************/ + /** + * @brief satisfies the TCSOP wait when the TCS itself reports arrival + * @details Spawned by move_to_target when TCS_AUTO_ONTARGET is enabled and + * lives only while that wait is open (auto_ontarget_active). + * Acknowledges when two consecutive 1 Hz reads show + * ?ONTARGET==1 AND motion tracking AND |telescope-commanded| < + * tcs_auto_ontarget_sep. All three predicates are required: + * the flag alone is position-agnostic, the separation alone + * cannot see a sub-threshold hop in progress, and motion alone + * says nothing about settling. The coords are the SCOPE-frame + * position move_to_target actually commanded. A wrong + * acknowledgment is self-limiting -- ACAM cannot solve a field + * that is not there and the acquisition retry machinery + * handles the failure -- so the conditions make wrong acks + * rare, not impossible. The operator's press works unchanged; + * whichever arrives first wins. The acknowledgment + * deliberately does not go through ontarget(): that clears + * cancel_flag, and a concurrent cancel must always win + * against this automatic path. + */ + void Sequence::dothread_auto_ontarget( double tgt_ra_deg, double tgt_dec_deg ) { + const std::string function("Sequencer::Sequence::dothread_auto_ontarget"); + std::stringstream message; + + if ( std::isnan(tgt_ra_deg) || std::isnan(tgt_dec_deg) ) { + logwrite( function, "ERROR invalid commanded coords; auto-ontarget sits this wait out" ); + return; + } + + int consecutive = 0; + + while ( this->auto_ontarget_active.load() && + !this->cancel_flag.load() && !this->is_ontarget.load() ) { + + std::this_thread::sleep_for( std::chrono::seconds(1) ); + + // the TCS's own settled flag (0|1), returned verbatim + // + std::string reply; + if ( this->tcsd.send( TCSD_NATIVE+" ?ONTARGET", reply ) != NO_ERROR ) { consecutive=0; continue; } + bool flag_on = ( reply.rfind( "1", 0 ) == 0 ); + + // motion state and separation from the cached 1 Hz tcsd telemetry + // + bool fresh = false; + bool tracking = false; + double sep_as = NAN; + { + std::lock_guard lock( tcsd_mtx ); + fresh = this->tcsinfo.is_fresh(); + if ( fresh ) { + tracking = ( this->tcsinfo.motion == TCS_MOTION_TRACKING_STR ); + double dra_deg = this->tcsinfo.ra_h * 15.0 - tgt_ra_deg; + dra_deg = std::fmod( dra_deg + 540.0, 360.0 ) - 180.0; // shortest arc across RA wrap + double dra_as = dra_deg * cos( tgt_dec_deg * M_PI / 180.0 ) * 3600.0; + double ddec_as = ( this->tcsinfo.dec_d - tgt_dec_deg ) * 3600.0; + sep_as = std::hypot( dra_as, ddec_as ); + } + } + + if ( !fresh || !flag_on || !tracking || + !( sep_as < this->tcs_auto_ontarget_sep ) ) { consecutive=0; continue; } + + if ( ++consecutive < 2 ) continue; // debounce: two reads in a row + + // Re-check at the last instant; do NOT clear cancel_flag (see @details). + // + if ( !this->auto_ontarget_active.load() || this->cancel_flag.load() ) return; + + message.str(""); message << "TCS reports on-target (?ONTARGET=1, tracking, sep=" + << std::fixed << std::setprecision(1) << sep_as + << " arcsec x2) -- sending ontarget"; + this->broadcast.notice( function, message.str() ); + this->is_ontarget.store( true ); + this->cv.notify_all(); + return; + } + } + /***** Sequencer::Sequence::dothread_auto_ontarget **************************/ + + /***** Sequencer::Sequence::move_to_target **********************************/ /** * @brief send request to TCS to move to target coordinates @@ -2297,10 +2379,24 @@ namespace Sequencer { this->broadcast.notice( function, "waiting for TCS operator to send \"ontarget\" signal" ); + // AUTO-ONTARGET (config TCS_AUTO_ONTARGET, default no): the TCS itself can + // satisfy this wait through the same ontarget() the operator's press uses. + // The watcher lives exactly as long as this wait (auto_ontarget_active). + // + this->auto_ontarget_active.store( this->tcs_auto_ontarget ); + if ( this->tcs_auto_ontarget ) { + // ra_out/dec_out are the SCOPE-frame coordinates this function actually + // commanded, which is the position REQPOS telemetry will report -- the + // database coordinates differ by the pointmode focal-plane offset. + std::thread( &Sequencer::Sequence::dothread_auto_ontarget, this, + ra_out, dec_out ).detach(); + } + while ( !this->cancel_flag.load() && !this->is_ontarget.load() ) { std::unique_lock lock(cv_mutex); this->cv.wait( lock, [this]() { return( this->is_ontarget.load() || this->cancel_flag.load() ); } ); } + this->auto_ontarget_active.store( false ); this->broadcast.notice( function, "received " +(this->cancel_flag.load() ? std::string("cancel") : std::string("ontarget")) diff --git a/sequencerd/sequence.h b/sequencerd/sequence.h index 8df8f33b..e7a53023 100644 --- a/sequencerd/sequence.h +++ b/sequencerd/sequence.h @@ -399,6 +399,12 @@ namespace Sequencer { this->is_ontarget.store(false); } + /** @brief satisfies the TCSOP wait when the TCS itself reports arrival + * (config TCS_AUTO_ONTARGET); coords are the commanded SCOPE + * frame position in degrees; see sequence.cpp */ + void dothread_auto_ontarget( double tgt_ra_deg, double tgt_dec_deg ); + std::atomic auto_ontarget_active{false}; ///< true only while move_to_target's TCSOP wait is open + /** @brief sends usercontinue signal, clears after 3 seconds */ inline void usercontinue() { this->cancel_flag.store(false); @@ -421,6 +427,8 @@ namespace Sequencer { double tcs_domeazi_ready; ///< max degrees azimuth that dome and telescope can differ before ready to observe double tcs_preauth_time; ///< seconds before end of exposure to notify TCS of next target's coords (0 to disable) double offset_settle_sec; ///< sec to wait after a target offset before exposing (config OFFSET_SETTLE_SEC; 0 disables) + bool tcs_auto_ontarget = false; ///< TCS ?ONTARGET may satisfy the TCSOP wait (config TCS_AUTO_ONTARGET; default no) + double tcs_auto_ontarget_sep = 5.0; ///< max |telescope-target| arcsec for auto-ontarget (config TCS_AUTO_ONTARGET_SEP) /// std::mutex tcs_ontarget_mtx; /// std::condition_variable tcs_ontarget_cv; diff --git a/sequencerd/sequencer_server.cpp b/sequencerd/sequencer_server.cpp index 405b43f3..56ea79d7 100644 --- a/sequencerd/sequencer_server.cpp +++ b/sequencerd/sequencer_server.cpp @@ -362,6 +362,31 @@ namespace Sequencer { applied++; } + // TCS_AUTO_ONTARGET + if (config.param[entry] == "TCS_AUTO_ONTARGET") { + this->sequence.tcs_auto_ontarget = ( config.arg[entry] == "yes" || + config.arg[entry] == "true" || + config.arg[entry] == "1" ); + message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; + logwrite( function, message.str() ); + applied++; + } + + // TCS_AUTO_ONTARGET_SEP + if (config.param[entry] == "TCS_AUTO_ONTARGET_SEP") { + try { + this->sequence.tcs_auto_ontarget_sep = std::stod( config.arg[entry] ); + } + catch (const std::exception &e) { + message.str(""); message << "ERROR parsing TCS_AUTO_ONTARGET_SEP: " << e.what(); + logwrite( function, message.str() ); + return ERROR; + } + message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; + logwrite( function, message.str() ); + applied++; + } + // OFFSET_SETTLE_SEC if (config.param[entry] == "OFFSET_SETTLE_SEC") { try { diff --git a/tcsd/tcs_interface.cpp b/tcsd/tcs_interface.cpp index 1fe8bee7..c05d566c 100644 --- a/tcsd/tcs_interface.cpp +++ b/tcsd/tcs_interface.cpp @@ -1857,6 +1857,9 @@ namespace TCS { if ( cmd == "?NAME" || cmd == "LAMPS?" || cmd == "NPS" || + cmd == "?ONTARGET" || // reply IS the flag (0|1), not a status code: + // 0 collides with TCS_SUCCESS ("success") and + // 1 falls through to "tcs_undefined" ERROR cmd == "?PARALLACTIC" || cmd == "?WEATHER" || cmd == "RAWDEC" || From a972af5fb4065fff35fb02a503559441fd33a437 Mon Sep 17 00:00:00 2001 From: Christoffer Fremling Date: Thu, 29 Oct 2026 16:06:07 +0200 Subject: [PATCH 06/10] sequencerd: pick the next target when the last readout starts When the last sub-exposure of a science target starts reading out, the target is completed and the next one is picked through the normal path (get_next, activate, publish, slew, acquire) while camerad reads out. A readout gate holds every camerad command for the new target (camera settings, expose) until camerad reports READY. There is no early pick in single-step or single-target mode, after a cancel, with a thread error set, or when the current row is a calibration. Readouts are counted from camerad's PIXELCOUNT messages, so the count cannot run ahead of the shutter; a PIXELCOUNT from the tail of the previous readout cannot end the next target's EXPOSE wait. camera_set runs in parallel with the slew and acquisition behind the readout gate and is joined just before the exposure; a target is never exposed unless its own camera settings were applied. The flexure compensator is set just before each exposure, re-applied during the readout between sub-exposures, and before a REPEAT exposes. Co-Authored-By: Claude Opus 5.5 --- sequencerd/sequence.cpp | 441 +++++++++++++++++++++++++++++++++++++--- sequencerd/sequence.h | 28 ++- 2 files changed, 437 insertions(+), 32 deletions(-) diff --git a/sequencerd/sequence.cpp b/sequencerd/sequence.cpp index 864048f7..19bac252 100644 --- a/sequencerd/sequence.cpp +++ b/sequencerd/sequence.cpp @@ -51,6 +51,25 @@ namespace Sequencer { if (jmessage.contains(Key::Camerad::READY)) { int isready = jmessage[Key::Camerad::READY].get(); this->can_expose.store(isready); + // The first READY after the next target was picked during a readout is + // the end of that readout (camerad reported READY=false when the + // exposure started and publishes in order): open the readout gate. + if ( isready && this->readout_gate_armed ) { + this->readout_gate_armed = false; + this->wait_state_manager.clear( Sequencer::SEQ_WAIT_READOUT ); + } + } + // Readout counting, step 1 of 2 (see the bookkeeping notes in sequence.h): + // the shutter report going from open to closed, in a report that is not + // PAUSED (camerad closes the shutter to pause, and says so), means an + // integration has ended. Open or paused reports disarm. + if ( jmessage.contains(Key::Camerad::SHUTTEROPEN) && jmessage[Key::Camerad::SHUTTEROPEN].is_boolean() ) { + const bool open = jmessage[Key::Camerad::SHUTTEROPEN].get(); + const bool paused = jmessage.contains(Key::Camerad::PAUSED) && jmessage[Key::Camerad::PAUSED].is_boolean() + && jmessage[Key::Camerad::PAUSED].get(); + if ( open || paused ) this->readout_armed = false; + else if ( this->last_shutteropen ) this->readout_armed = true; + this->last_shutteropen = open; } this->camerad_cv.notify_all(); } @@ -512,10 +531,47 @@ namespace Sequencer { // Set READOUT flag and clear EXPOSE flag when pixels start coming in // ------------------------------------------------------------------ // - if ( seq.arm_readout_flag && statstr.compare( 0, 10, "PIXELCOUNT" )==0 ) { // async message tag PIXELCOUNT - seq.arm_readout_flag = false; - seq.wait_state_manager.set_and_clear( {Sequencer::SEQ_WAIT_READOUT}, // set READOUT - {Sequencer::SEQ_WAIT_EXPOSE} ); // clear EXPOSE + if ( statstr.compare( 0, 10, "PIXELCOUNT" )==0 ) { // async message tag PIXELCOUNT + // How far into its readout this device is: "PIXELCOUNT_: + // " (the first two numbers after the colon are + // used; -1 if they cannot be read). + double readout_fraction = -1.0; + const auto colon = statstr.find( ':' ); + if ( colon != std::string::npos ) { + std::istringstream fields( statstr.substr( colon+1 ) ); + std::string token; + std::vector numbers; + while ( numbers.size() < 2 && fields >> token ) { + try { + std::size_t used = 0; + const double value = std::stod( token, &used ); + if ( used == token.size() ) numbers.push_back( value ); + } + catch ( const std::exception & ) { } + } + if ( numbers.size() == 2 && numbers[1] > 0.0 ) readout_fraction = numbers[0] / numbers[1]; + } + // The first PIXELCOUNT of a readout ends the EXPOSE wait -- but not one + // from the second half of a readout: that is the tail of the previous + // target's readout, arriving after the next expose command. + if ( seq.arm_readout_flag && !( readout_fraction >= 0.5 ) ) { + seq.arm_readout_flag = false; + seq.wait_state_manager.set_and_clear( {Sequencer::SEQ_WAIT_READOUT}, // set READOUT + {Sequencer::SEQ_WAIT_EXPOSE} ); // clear EXPOSE + } + // Readout counting, step 2 of 2 (step 1 in handletopic_camerad): an + // armed count is completed by a PIXELCOUNT from the first half of a + // readout. A message from the tail of an earlier readout, or one that + // cannot be read, completes nothing; a count completed here leaves at + // least half a readout before camerad can open the shutter again. + { + std::lock_guard lock(seq.camerad_mtx); + if ( seq.readout_armed && readout_fraction >= 0.0 && readout_fraction < 0.5 ) { + seq.readout_armed = false; + seq.readouts_started++; + seq.camerad_cv.notify_all(); + } + } } // --------------------- @@ -640,6 +696,16 @@ namespace Sequencer { this->is_ontarget.store(false); this->is_usercontinue.store(false); + // On every way out of this function except a cancel, do not report the + // sequence finished while camerad is still reading out a target that was + // completed at the start of its last readout. Declared after the + // ScopedStates above, so it runs before they set READY. + // + struct ReadoutGateOnExit { + Sequence &seq; + ~ReadoutGateOnExit() { try { seq.wait_for_readout_gate( "ending the sequence" ); } catch (...) { } } + } readout_gate_on_exit{ *this }; + // This is the main loop which runs as long as there are targets in the database, // or until cancelled. Or, this will run once in the case of single-target-mode, // which is invoked by storing an OBSID in the class variable single_obsid. @@ -696,6 +762,23 @@ namespace Sequencer { } this->publish_targetinfo(); // publish the now-active target + + // A target picked while the previous one is still reading out starts + // at once only if the telescope has to slew to it: a science row with + // coordinates and SLIT pointing. Anything else -- a calibration, a row + // without coordinates, ACAM pointing -- starts when that readout has + // finished, exactly as before. It is already 'active' while it waits, + // like every picked row. + // + { + const bool slews = !this->target.iscal + && !( this->target.ra_hms.empty() && this->target.dec_dms.empty() ) + && !( this->target.pointmode == Acam::POINTMODE_ACAM ); + if ( !slews && !this->wait_for_readout_gate( "starting "+std::string(this->target.name) ) ) { + this->broadcast.notice( function, "sequence cancelled" ); + return; + } + } } else // targetstate not TARGET_FOUND if ( targetstate == TargetInfo::TARGET_NOT_FOUND ) { // no target found is an automatic stop @@ -717,6 +800,14 @@ namespace Sequencer { this->is_repeat_target.store( this->repeat_target() ); if ( this->is_repeat_target.load() ) this->broadcast.notice( function, "repeat target" ); + // Flexure compensation is computed from the telescope position at the + // moment flexured applies it. When the telescope stays where it is (a + // calibration, or a repeat of the same pointing) it is set with the other + // workers, in parallel, as it always was. When the telescope slews, that + // would compensate for the PREVIOUS pointing, so it is set after + // acquisition instead, just before the exposure. + const bool flexure_after_acquisition = !this->target.iscal && !this->is_repeat_target.load(); + // threads to start, pair their ThreadStatusBit with the function to call std::vector>> worker_threads; @@ -739,15 +830,18 @@ namespace Sequencer { // threads to start, pair their ThreadStatusBit with the function to call // + // camera_set is not here: it talks to camerad, so it runs just before + // the exposure, behind the readout gate (see "starting exposure"). worker_threads = { { THR_MOVE_TO_TARGET, std::bind(&Sequence::move_to_target, this) }, - { THR_CAMERA_SET, std::bind(&Sequence::camera_set, this) }, { THR_FOCUS_SET, std::bind(&Sequence::focus_set, this) }, - { THR_FLEXURE_SET, std::bind(&Sequence::flexure_set, this) }, { THR_CALIB_SET, std::bind(&Sequence::calib_set, this) }, // for CAL targets, slit comes from database, otherwise use VSM acquire position { THR_SLIT_SET, std::bind(&Sequence::slit_set, this, this->target.iscal ? Sequencer::VSM_EXPOSE : Sequencer::VSM_ACQUIRE) } }; + if ( !flexure_after_acquisition ) { // telescope does not move: set it now, in parallel + worker_threads.push_back( { THR_FLEXURE_SET, std::bind(&Sequence::flexure_set, this) } ); + } } // pair their ThreadStatusBit with their future @@ -759,6 +853,19 @@ namespace Sequencer { worker_futures.emplace_back( thr, std::async(std::launch::async, func) ); } + // camera_set talks to camerad, so it waits for the readout gate, then runs + // in parallel with everything else (a channel power-up takes seconds) and + // is joined just before the exposure. Not for ACAM pointing, which never + // exposes here. + // + std::future camera_future; + if ( !( this->target.pointmode == Acam::POINTMODE_ACAM ) ) { + camera_future = std::async( std::launch::async, [this]() -> long { + if ( !this->wait_for_readout_gate( "setting the camera for "+std::string(this->target.name) ) ) return ABORT; + return this->camera_set(); + } ); + } + // wait for the threads to complete. these can be cancelled. // for ( auto &[thr, future] : worker_futures) { @@ -830,6 +937,14 @@ namespace Sequencer { } } + // For a target the telescope slewed to (flexure_after_acquisition), the + // flexure compensation is set here -- telescope on target, acquisition + // and target offset done -- in parallel with the slit move and the + // offset settle, and joined before the expose command. For NEXP>1 it is + // re-applied during each readout (flexure_reapply, in the readout wait). + // + std::future flexure_future; + if ( !this->target.iscal ) { // send offsets only on fineacquire, otherwise user needs to fix things if ( !this->is_repeat_target.load() && @@ -840,6 +955,9 @@ namespace Sequencer { return; } } + if ( flexure_after_acquisition ) { + flexure_future = std::async( std::launch::async, &Sequence::flexure_set, this ); + } // ensure slit offset is in "expose" position when needed try { error |= this->slit_set(Sequencer::VSM_EXPOSE); @@ -861,6 +979,53 @@ namespace Sequencer { std::this_thread::sleep_for( std::chrono::duration( this->offset_settle_sec ) ); } + // The compensator must be set before the shutter opens. A failure keeps + // the semantics flexure_set has in the worker set: the sequence stops. + // + if ( flexure_future.valid() ) { + try { + long flexure_error = flexure_future.get(); + error |= flexure_error; + if ( flexure_error != NO_ERROR ) { + this->broadcast.error( function, Sequencer::thread_names.at(THR_FLEXURE_SET)+" failed" ); + } + } + catch (const std::exception& e) { + this->broadcast.error( function, Sequencer::thread_names.at(THR_FLEXURE_SET)+" : "+std::string(e.what()) ); + return; + } + } + + // Nothing reaches camerad before the previous target's readout has + // finished: when this target was picked at the start of that readout, + // the camera settings (camera_future) and the expose wait for READY. + // + if ( !this->wait_for_readout_gate( "exposing "+std::string(this->target.name) ) ) { + this->broadcast.notice( function, "sequence cancelled" ); + return; + } + if ( camera_future.valid() ) { + try { + long camera_error = camera_future.get(); + error |= camera_error; + // never expose unless this target's camera settings were applied + if ( camera_error != NO_ERROR ) { + this->broadcast.error( function, Sequencer::thread_names.at(THR_CAMERA_SET) + +( camera_error==ABORT ? " cancelled" : " failed" )+": not exposing" ); + return; + } + this->broadcast.notice( function, Sequencer::thread_names.at(THR_CAMERA_SET)+" ready" ); + } + catch (const std::exception& e) { + this->broadcast.error( function, Sequencer::thread_names.at(THR_CAMERA_SET)+" : "+std::string(e.what()) ); + return; + } + } + if ( this->cancel_flag.load() ) { + this->broadcast.notice( function, "sequence cancelled" ); + return; + } + logwrite( function, "starting exposure" ); ///< TODO @todo log to telemetry! // Start the exposure in a thread... @@ -907,19 +1072,62 @@ namespace Sequencer { // of the last exposure finishes — the correct completion signal for both // single and multi-exposure sequences. Skip the wait for frame transfer. // + // While waiting, every counted sub-exposure readout (readouts_started; see + // the bookkeeping notes in sequence.h) is acted on once: a readout that more sub-exposures + // follow gets the flexure compensation re-applied during it. When the + // LAST one starts reading out, this target is complete -- its + // integration has ended, and camerad took its header telemetry at + // exposure start and AIRMASS1/MJD1 at shutter close -- so the loop + // completes it and picks the next target now instead of after READY, + // unless single-step, a single target, a cancel or a thread error says + // otherwise. The readout gate keeps camerad untouched until READY. + // + bool picked_during_readout = false; if (!this->is_science_frame_transfer) { logwrite( function, "waiting for readout" ); + int readouts_handled = 0; std::unique_lock lock(this->camerad_mtx); - while ( !this->camerad_cv.wait_for( lock, std::chrono::seconds(30), - [this]() { return this->can_expose.load() || this->cancel_flag.load(); } ) ) { - logwrite( function, "waiting for readout — requesting snapshot" ); + while ( true ) { + const bool signalled = this->camerad_cv.wait_for( lock, std::chrono::seconds(30), + [this,&readouts_handled]() { return this->can_expose.load() || this->cancel_flag.load() || + this->readouts_started > readouts_handled; } ); + if ( this->can_expose.load() || this->cancel_flag.load() ) break; + if ( !signalled ) { + logwrite( function, "waiting for readout — requesting snapshot" ); + lock.unlock(); + this->request_snapshot(); + lock.lock(); + continue; + } + const int sub = ++readouts_handled; // readout of sub-exposure number sub (1-based) has begun + const int expected = this->readouts_expected; // the N sent to camerad, not the live row + if ( sub == expected && !this->target.iscal && !this->do_once.load() && this->single_obsid.empty() && + !this->cancel_flag.load() && !this->thread_error_manager.are_any_set() ) { + this->readout_gate_armed = true; // under camerad_mtx: no READY can slip in between + picked_during_readout = true; + break; + } lock.unlock(); - this->request_snapshot(); + if ( sub < expected ) { + this->flexure_reapply( sub, expected ); + } + else if ( sub > expected ) { + message.str(""); message << "readout " << sub << " seen for a " << expected + << "-exposure command: camerad is running more sub-exposures than were sent"; + this->broadcast.warning( function, message.str() ); + } lock.lock(); } - this->broadcast.notice( function, "done waiting for readout" ); + if ( picked_during_readout ) { + this->broadcast.notice( function, "last readout of "+std::string(this->target.name) + +" under way: completing it and picking the next target" ); + } + else { + this->broadcast.notice( function, "done waiting for readout" ); + } } - this->wait_state_manager.clear( Sequencer::SEQ_WAIT_READOUT ); + // after a pick the READOUT bit stays set until camerad's READY opens the gate + if ( !picked_during_readout ) this->wait_state_manager.clear( Sequencer::SEQ_WAIT_READOUT ); // Now that we're done waiting, check for errors or abort // @@ -996,9 +1204,13 @@ namespace Sequencer { if (this->cancel_flag.load()) { logwrite(function, "sequence cancelled"); - return NO_ERROR; + return ABORT; // nothing was set: the caller must not expose } } + // camerad_mtx guards can_expose only; holding it across the camerad + // commands below would block the ZMQ subscriber thread (every topic) for + // their duration -- a channel power-up takes seconds + lock.unlock(); logwrite( function, "setting camera parameters"); @@ -2170,7 +2382,7 @@ namespace Sequencer { * cancel_flag, and a concurrent cancel must always win * against this automatic path. */ - void Sequence::dothread_auto_ontarget( double tgt_ra_deg, double tgt_dec_deg ) { + void Sequence::dothread_auto_ontarget( double tgt_ra_deg, double tgt_dec_deg, unsigned gen ) { const std::string function("Sequencer::Sequence::dothread_auto_ontarget"); std::stringstream message; @@ -2181,7 +2393,10 @@ namespace Sequencer { int consecutive = 0; - while ( this->auto_ontarget_active.load() && + // gen retires this watcher the moment a newer one is launched, so one + // still sleeping from an earlier move can never acknowledge a new one. + // + while ( this->auto_ontarget_active.load() && this->auto_ontarget_gen.load() == gen && !this->cancel_flag.load() && !this->is_ontarget.load() ) { std::this_thread::sleep_for( std::chrono::seconds(1) ); @@ -2215,16 +2430,25 @@ namespace Sequencer { if ( ++consecutive < 2 ) continue; // debounce: two reads in a row - // Re-check at the last instant; do NOT clear cancel_flag (see @details). + // Re-check at the last instant and acknowledge under ontarget_ack_mtx, so + // a new move (which retires watchers and clears is_ontarget under the + // same lock) cannot fall between the check and the store. Do NOT clear + // cancel_flag (see @details). cv_mutex is taken before the notify so a + // waiter between its predicate check and its wait cannot miss it. // - if ( !this->auto_ontarget_active.load() || this->cancel_flag.load() ) return; + { + std::lock_guard lock( this->ontarget_ack_mtx ); + if ( !this->auto_ontarget_active.load() || this->auto_ontarget_gen.load() != gen || + this->cancel_flag.load() ) return; + this->is_ontarget.store( true ); + } + { std::lock_guard lock( this->cv_mutex ); } + this->cv.notify_all(); message.str(""); message << "TCS reports on-target (?ONTARGET=1, tracking, sep=" << std::fixed << std::setprecision(1) << sep_as << " arcsec x2) -- sending ontarget"; this->broadcast.notice( function, message.str() ); - this->is_ontarget.store( true ); - this->cv.notify_all(); return; } } @@ -2268,6 +2492,7 @@ namespace Sequencer { return NO_ERROR; } + error = NO_ERROR; // Before sending target coordinates to TCS, @@ -2342,6 +2567,16 @@ namespace Sequencer { decimal_to_sexa( dec_out, decstr ); message.str(""); message << "[DEBUG] moving to SCOPE COORDS= " << rastr << " " << decstr << " " << angle_out << " J2000"; logwrite( function, message.str() ); + // Retire every auto-ontarget watcher still alive (one from an earlier + // move) and clear the flag, in one step with respect to + // a watcher's acknowledgment: only an ack of THIS move may satisfy the + // wait below. + unsigned ontarget_gen = 0; + { + std::lock_guard lock( this->ontarget_ack_mtx ); + ontarget_gen = ++this->auto_ontarget_gen; + this->is_ontarget.store(false); + } error = this->tcsd.send( coords_cmd.str(), coords_reply ); // send to the TCS // if not success then wait 1s and try again @@ -2388,8 +2623,9 @@ namespace Sequencer { // ra_out/dec_out are the SCOPE-frame coordinates this function actually // commanded, which is the position REQPOS telemetry will report -- the // database coordinates differ by the pointmode focal-plane offset. + // ontarget_gen was taken, retiring older watchers, before the COORDS. std::thread( &Sequencer::Sequence::dothread_auto_ontarget, this, - ra_out, dec_out ).detach(); + ra_out, dec_out, ontarget_gen ).detach(); } while ( !this->cancel_flag.load() && !this->is_ontarget.load() ) { @@ -2420,6 +2656,41 @@ namespace Sequencer { /***** Sequencer::Sequence::move_to_target **********************************/ + /***** Sequencer::Sequence::wait_for_readout_gate ***************************/ + /** + * @brief wait until the previous target's readout has finished + * @param[in] before what is waiting, for the log + * @return true to proceed, false if cancelled while waiting + * + * The gate is armed when the next target is picked at the start of the + * previous target's last readout (sequence_start) and opened by the first + * camerad READY report after that (handletopic_camerad). Returns at once + * when it is not armed, which is every case but that one. + */ + bool Sequence::wait_for_readout_gate( const std::string &before ) { + const std::string function("Sequencer::Sequence::wait_for_readout_gate"); + std::unique_lock lock( this->camerad_mtx ); + if ( !this->readout_gate_armed ) return true; + logwrite( function, "previous target still reading out: waiting before "+before ); + while ( this->readout_gate_armed && !this->cancel_flag.load() ) { + if ( !this->camerad_cv.wait_for( lock, std::chrono::seconds(30), + [this]() { return !this->readout_gate_armed || this->cancel_flag.load(); } ) ) { + logwrite( function, "still waiting for the previous readout -- requesting snapshot" ); + lock.unlock(); + this->request_snapshot(); + lock.lock(); + } + } + if ( this->readout_gate_armed ) { + logwrite( function, "cancelled while waiting for the previous readout" ); + return false; + } + logwrite( function, "previous readout finished" ); + return true; + } + /***** Sequencer::Sequence::wait_for_readout_gate ***************************/ + + /***** Sequencer::Sequence::dothread_notify_tcs *****************************/ /** * @brief notify TCS operator of next target coords for preauthorization, no move @@ -2548,31 +2819,110 @@ namespace Sequencer { return NO_ERROR; } - // build up list of activate chans + // the compensate command names this target's active channels + const std::string comp = this->flexure_compensate_command(); + const std::string activechans = comp.substr( std::string(FLEXURED_COMPENSATE).size() ); + + if ( !this->cancel_flag.load() && + this->flexured.command( comp ) != NO_ERROR ) { + this->broadcast.error( function, "setting flexure compensator" ); + this->thread_error_manager.set( THR_FLEXURE_SET ); + throw std::runtime_error("setting flexure compensator"); + } + + this->is_flexure_default.store(false); + + this->broadcast.notice( function, "flexure compensator set for "+activechans); + + return NO_ERROR; + } + /***** Sequencer::Sequence::flexure_set *************************************/ + + + /***** Sequencer::Sequence::flexure_compensate_command **********************/ + /** + * @brief the FLEXURED_COMPENSATE command for the current target + * @return "compensate [ ...]" with the channels active for + * the target's calibration entry ("SCIENCE" for science targets) + */ + std::string Sequence::flexure_compensate_command() { std::ostringstream activechans; const std::string calname = std::string(this->target.iscal ? this->target.name : "SCIENCE"); const auto &calinfo = this->caltarget.get_info(calname); for (const auto &[chan,active] : calinfo.channel_active) { if (active) activechans << " " << chan; } + return std::string(FLEXURED_COMPENSATE) + activechans.str(); + } + /***** Sequencer::Sequence::flexure_compensate_command **********************/ - std::ostringstream comp; - comp << FLEXURED_COMPENSATE << activechans.str(); - if ( !this->cancel_flag.load() && - this->flexured.command( comp.str() ) != NO_ERROR ) { - this->broadcast.error( function, "setting flexure compensator" ); - this->thread_error_manager.set( THR_FLEXURE_SET ); - throw std::runtime_error("setting flexure compensator"); + /***** Sequencer::Sequence::flexure_reapply *********************************/ + /** + * @brief re-issue the flexure compensation between two sub-exposures + * @param[in] sub the sub-exposure (1-based) whose readout is in progress + * @return ERROR | NO_ERROR (informational only) + * + * Called by sequence_start during the readout of sub-exposure sub when + * sub-exposure sub+1 of expected follows. flexured recomputes from the + * telescope position at this moment, so the next sub-exposure starts + * corrected. Skipped (the compensation set at the start of the series + * stays) unless that readout is still running and the next shutter has not + * opened, the TCS telemetry is fresh and finite (flexured itself does not + * check the values), and no thread error is set. A failure is reported and + * otherwise ignored: nothing may disturb a running exposure command. + */ + long Sequence::flexure_reapply( int sub, int expected ) { + const std::string function("Sequencer::Sequence::flexure_reapply"); + std::stringstream message; + + if ( !this->should_flexcomp.load() || this->cancel_flag.load() ) return NO_ERROR; + + auto skipped = [&]( const std::string &why ) { + message.str(""); message << "flexure compensation NOT re-applied before exposure " << sub+1 + << " of " << expected << ": " << why; + this->broadcast.warning( function, message.str() ); + return ERROR; + }; + auto next_started = [this, sub]() { + std::lock_guard lock( this->camerad_mtx ); + return ( this->last_shutteropen || this->readouts_started != sub ); + }; + + if ( this->thread_error_manager.are_any_set() ) return skipped( "a thread error is set" ); + { + std::lock_guard lock( this->tcsd_mtx ); + if ( !this->daemon_manager.is_set( Sequencer::DAEMON_TCS ) || !this->tcsinfo.is_fresh() || + !std::isfinite( this->tcsinfo.zenithangle ) || !std::isfinite( this->tcsinfo.cassangle ) || + !std::isfinite( this->tcsinfo.pa ) ) { + return skipped( "TCS telemetry not fresh or not valid" ); + } } + if ( next_started() ) return skipped( "the next exposure has already started" ); + + const std::string comp = this->flexure_compensate_command(); + + message.str(""); message << "re-applying flexure compensation during readout " << sub << " of " + << expected << " for " << this->target.name; + logwrite( function, message.str() ); + + if ( this->flexured.command( comp ) != NO_ERROR ) return skipped( "flexured returned an error" ); this->is_flexure_default.store(false); - this->broadcast.notice( function, "flexure compensator set for "+activechans.str()); + if ( next_started() ) { + message.str(""); message << "flexure compensator re-applied, but exposure " << sub+1 << " of " << expected + << " had already started: the actuators moved during its first seconds"; + this->broadcast.warning( function, message.str() ); + return ERROR; + } + message.str(""); message << "flexure compensator re-applied before exposure " << sub+1 + << " of " << expected; + this->broadcast.notice( function, message.str() ); return NO_ERROR; } - /***** Sequencer::Sequence::flexure_set *************************************/ + /***** Sequencer::Sequence::flexure_reapply *********************************/ /***** Sequencer::Sequence::calib_set ***************************************/ @@ -2784,7 +3134,9 @@ namespace Sequencer { // so this loop exits after at most a few iterations. // if ( abort_during_run ) { - const auto drain_timeout = std::chrono::steady_clock::now() + std::chrono::seconds(5); + // 20 s: the readout wait may be inside a flexured re-apply (client + // timeout 6 s + 2 s, flexured itself up to ~11 s) when the cancel lands + const auto drain_timeout = std::chrono::steady_clock::now() + std::chrono::seconds(20); while ( this->thread_state_manager.is_set( Sequencer::THR_SEQUENCE_START ) && std::chrono::steady_clock::now() < drain_timeout ) { std::this_thread::sleep_for( std::chrono::milliseconds(20) ); @@ -2893,6 +3245,8 @@ namespace Sequencer { this->is_ontarget.store(false); this->is_usercontinue.store(false); + if ( !this->wait_for_readout_gate( "repeating the exposure" ) ) return ERROR; + std::string targetstatus; this->target.get_specified_target( this->prev_single_obsid, targetstatus ); @@ -2936,6 +3290,24 @@ namespace Sequencer { return NO_ERROR; } + // Flexure for the telescope position now. Only after camera_set has seen + // camerad ready: a REPEAT is accepted while the previous one is still + // integrating, and the actuators must not move under it. + // + try { + if ( this->flexure_set() != NO_ERROR ) { + this->broadcast.error( function, Sequencer::thread_names.at(THR_FLEXURE_SET)+" failed" ); + } + } + catch (const std::exception& e) { + this->broadcast.error( function, Sequencer::thread_names.at(THR_FLEXURE_SET)+" exception: "+std::string(e.what()) ); + return ERROR; + } + if ( this->cancel_flag.load() ) { + this->broadcast.notice(function, "cancelled repeat exposure"); + return NO_ERROR; + } + // Start the exposure in a thread... // auto start_exposure = std::async(std::launch::async, &Sequence::trigger_exposure, this); @@ -2986,6 +3358,13 @@ namespace Sequencer { // if ( this->tcs_preauth_time > 0 ) this->notify_tcs_next_target = true; else this->notify_tcs_next_target = false; + { + std::lock_guard lock(this->camerad_mtx); + this->last_shutteropen = false; // readout bookkeeping starts fresh with this expose command; + this->readout_armed = false; // the shutter is closed until camerad opens it for this one + this->readouts_started = 0; + this->readouts_expected = this->target.nexp; // the N sent below, fixed for this command + } this->arm_readout_flag = true; // enables the async_listener to look for the readout and clear the EXPOSE bit this->set_imgtype(); diff --git a/sequencerd/sequence.h b/sequencerd/sequence.h index e7a53023..9e6e7e13 100644 --- a/sequencerd/sequence.h +++ b/sequencerd/sequence.h @@ -402,8 +402,10 @@ namespace Sequencer { /** @brief satisfies the TCSOP wait when the TCS itself reports arrival * (config TCS_AUTO_ONTARGET); coords are the commanded SCOPE * frame position in degrees; see sequence.cpp */ - void dothread_auto_ontarget( double tgt_ra_deg, double tgt_dec_deg ); + void dothread_auto_ontarget( double tgt_ra_deg, double tgt_dec_deg, unsigned gen ); std::atomic auto_ontarget_active{false}; ///< true only while move_to_target's TCSOP wait is open + std::atomic auto_ontarget_gen{0}; ///< launch generation; a watcher retires when a newer one is launched + std::mutex ontarget_ack_mtx; ///< a watcher's acknowledgment vs a new move retiring watchers and clearing is_ontarget /** @brief sends usercontinue signal, clears after 3 seconds */ inline void usercontinue() { @@ -439,6 +441,27 @@ namespace Sequencer { std::condition_variable acam_cv; std::mutex camerad_mtx; std::condition_variable camerad_cv; + // Readout bookkeeping for the current expose command, guarded by camerad_mtx + // and reset in trigger_exposure(). A sub-exposure's readout is counted in two + // steps. (1) camerad's shutter report goes from open to closed in a report + // that is not PAUSED: that integration has ended, so the count is armed + // (handletopic_camerad). (2) A PIXELCOUNT from the first half of a readout + // completes it (async listener). Any open or paused report disarms. + // camerad's reports arrive in the order it published them, so a late or + // repeated report cannot arm twice for one close, and a late PIXELCOUNT + // cannot count a sub-exposure that is integrating. The count never exceeds + // the integrations that have ended; a lost report only leaves one uncounted. + bool last_shutteropen = false; ///< SHUTTEROPEN in the last camerad report since the expose command + bool readout_armed = false; ///< an integration ended (open -> closed, not paused), its readout not yet counted + int readouts_started = 0; ///< sub-exposure readouts counted since the expose command + int readouts_expected = 0; ///< N of that expose command (this->target can be rewritten by manual commands) + // The readout gate, guarded by camerad_mtx. When the last readout of a + // target starts, the loop completes that target and picks the next one + // at once; nothing may then reach camerad (camera settings, an expose) + // until camerad reports READY again. Armed at that pick, opened by the + // first READY=true report after it (handletopic_camerad), and held + // across a stop or restart until then. + bool readout_gate_armed = false; std::mutex slitd_mtx; std::condition_variable slitd_cv; std::mutex tcsd_mtx; @@ -627,6 +650,9 @@ namespace Sequencer { long camera_set(); ///< sets camera according to target entry params long slit_set(VirtualSlitMode mode=VSM_DATABASE); ///< sets slit according to target entry params and mode long move_to_target(); ///< sends request to TCS to move to target coords + bool wait_for_readout_gate( const std::string &before ); ///< wait for the previous target's readout to finish; false if cancelled + long flexure_reapply( int sub, int expected ); ///< re-issues the flexure compensation between sub-exposures (never fatal) + std::string flexure_compensate_command(); ///< FLEXURED_COMPENSATE plus the active channels for the current target static void dothread_notify_tcs( Sequencer::Sequence &seq ); ///< like move_to_target but for preauth only long focus_set(); long flexure_set(); From 60b31c09a77f9251f927935dc1aa9f92a932572b Mon Sep 17 00:00:00 2001 From: Christoffer Fremling Date: Thu, 29 Oct 2026 16:06:33 +0200 Subject: [PATCH 07/10] sequencerd: auto-ontarget on by default; require the cass ring at the commanded angle TCS_AUTO_ONTARGET defaults to yes. The automatic ontarget now also requires |TCS cass angle - RINGGO angle| < TCS_AUTO_ONTARGET_CASS (0.05 deg) on the same two consecutive reads: ACAM's goal is computed from the cass angle at the acquire, about 8.5" per degree of ring still to turn. When everything but the ring has held for 20 s, one warning tells the operator to press ontarget once the ring is in place. Co-Authored-By: Claude Opus 5.5 --- Config/sequencerd.cfg.in | 3 +- sequencerd/sequence.cpp | 57 +++++++++++++++++++++++++-------- sequencerd/sequence.h | 5 +-- sequencerd/sequencer_server.cpp | 15 +++++++++ 4 files changed, 63 insertions(+), 17 deletions(-) diff --git a/Config/sequencerd.cfg.in b/Config/sequencerd.cfg.in index d8db2de4..f0a5c079 100644 --- a/Config/sequencerd.cfg.in +++ b/Config/sequencerd.cfg.in @@ -150,8 +150,9 @@ TCS_OFFSET_RATE_RA=45 # TCS offset rate for RA ("MRATE") in arcse TCS_OFFSET_RATE_DEC=45 # TCS offset rate for DEC ("MRATE") in arcsec per second TCS_SETTLE_TIMEOUT=20 # seconds to wait for telescope to settle (for "TRACKING_STABLY") TCS_SETTLE_STABLE=1.5 # time in fractional seconds (to 0.1) that TCS must report TRACKING before it is really stable -TCS_AUTO_ONTARGET=no # yes: TCS ?ONTARGET=1 + tracking + sep < TCS_AUTO_ONTARGET_SEP (2 consecutive 1 Hz reads) satisfies the ontarget wait; operator press always works +TCS_AUTO_ONTARGET=yes # yes: TCS ?ONTARGET=1 + tracking + sep < TCS_AUTO_ONTARGET_SEP + cass ring within TCS_AUTO_ONTARGET_CASS (2 consecutive 1 Hz reads) satisfies the ontarget wait; operator press always works TCS_AUTO_ONTARGET_SEP=5.0 # arcsec: max |telescope-target| for auto-ontarget +TCS_AUTO_ONTARGET_CASS=0.05 # deg: max |TCS cass angle - commanded ring angle| for auto-ontarget TCS_DOMEAZI_READY=10 # max degrees azimuth that dome and telescope can differ before ready to observe TCS_PREAUTH_TIME=10 # seconds before end of exposure to notify TCS of next target's coords (0 to disable) diff --git a/sequencerd/sequence.cpp b/sequencerd/sequence.cpp index 19bac252..aa24ba62 100644 --- a/sequencerd/sequence.cpp +++ b/sequencerd/sequence.cpp @@ -2368,11 +2368,15 @@ namespace Sequencer { * lives only while that wait is open (auto_ontarget_active). * Acknowledges when two consecutive 1 Hz reads show * ?ONTARGET==1 AND motion tracking AND |telescope-commanded| < - * tcs_auto_ontarget_sep. All three predicates are required: - * the flag alone is position-agnostic, the separation alone - * cannot see a sub-threshold hop in progress, and motion alone - * says nothing about settling. The coords are the SCOPE-frame - * position move_to_target actually commanded. A wrong + * tcs_auto_ontarget_sep AND |TCS cass angle - commanded ring + * angle| < tcs_auto_ontarget_cass. All four are required: the + * flag alone is position-agnostic, the separation alone cannot + * see a sub-threshold hop in progress, motion alone says + * nothing about settling, and none of them covers the cass + * ring, which turns on its own after RINGGO (ACAM's goal is + * computed from the cass angle at the acquire, ~8.5" per + * degree of ring still to go). The coords and ring angle are + * what move_to_target actually commanded. A wrong * acknowledgment is self-limiting -- ACAM cannot solve a field * that is not there and the acquisition retry machinery * handles the failure -- so the conditions make wrong acks @@ -2382,16 +2386,18 @@ namespace Sequencer { * cancel_flag, and a concurrent cancel must always win * against this automatic path. */ - void Sequence::dothread_auto_ontarget( double tgt_ra_deg, double tgt_dec_deg, unsigned gen ) { + void Sequence::dothread_auto_ontarget( double tgt_ra_deg, double tgt_dec_deg, double tgt_cass_deg, unsigned gen ) { const std::string function("Sequencer::Sequence::dothread_auto_ontarget"); std::stringstream message; - if ( std::isnan(tgt_ra_deg) || std::isnan(tgt_dec_deg) ) { - logwrite( function, "ERROR invalid commanded coords; auto-ontarget sits this wait out" ); + if ( std::isnan(tgt_ra_deg) || std::isnan(tgt_dec_deg) || std::isnan(tgt_cass_deg) ) { + logwrite( function, "ERROR invalid commanded coords or ring angle; auto-ontarget sits this wait out" ); return; } int consecutive = 0; + int64_t ring_wait_since_us = 0; // when everything but the ring first held + bool ring_warned = false; // gen retires this watcher the moment a newer one is launched, so one // still sleeping from an earlier move can never acknowledge a new one. @@ -2412,6 +2418,8 @@ namespace Sequencer { bool fresh = false; bool tracking = false; double sep_as = NAN; + double dcass_deg = NAN; + double cass_now = NAN; { std::lock_guard lock( tcsd_mtx ); fresh = this->tcsinfo.is_fresh(); @@ -2422,11 +2430,31 @@ namespace Sequencer { double dra_as = dra_deg * cos( tgt_dec_deg * M_PI / 180.0 ) * 3600.0; double ddec_as = ( this->tcsinfo.dec_d - tgt_dec_deg ) * 3600.0; sep_as = std::hypot( dra_as, ddec_as ); + // the cass ring against the angle RINGGO was sent, either side of the wrap + cass_now = this->tcsinfo.cassangle; + dcass_deg = std::fabs( std::remainder( cass_now - tgt_cass_deg, 360.0 ) ); + } + } + + const bool all_but_ring = fresh && flag_on && tracking && ( sep_as < this->tcs_auto_ontarget_sep ); + const bool ring_ok = ( dcass_deg < this->tcs_auto_ontarget_cass ); + + // On target with only the ring missing for 20 s (a refused RINGGO, a ring + // parked off the command, an unreadable angle): say so once, so the + // operator knows the wait needs a press. + if ( all_but_ring && !ring_ok ) { + if ( ring_wait_since_us == 0 ) ring_wait_since_us = get_time_us(); + else if ( !ring_warned && get_time_us() - ring_wait_since_us > 20000000 ) { + ring_warned = true; + message.str(""); message << "telescope on target but the cass ring reads " << cass_now + << " deg, commanded " << tgt_cass_deg + << " deg: press ontarget once the ring is in place"; + this->broadcast.warning( function, message.str() ); + } } - } + else if ( !all_but_ring ) ring_wait_since_us = 0; - if ( !fresh || !flag_on || !tracking || - !( sep_as < this->tcs_auto_ontarget_sep ) ) { consecutive=0; continue; } + if ( !all_but_ring || !ring_ok ) { consecutive=0; continue; } if ( ++consecutive < 2 ) continue; // debounce: two reads in a row @@ -2447,7 +2475,8 @@ namespace Sequencer { message.str(""); message << "TCS reports on-target (?ONTARGET=1, tracking, sep=" << std::fixed << std::setprecision(1) << sep_as - << " arcsec x2) -- sending ontarget"; + << " arcsec, ring within " << std::setprecision(3) << dcass_deg + << " deg x2) -- sending ontarget"; this->broadcast.notice( function, message.str() ); return; } @@ -2614,7 +2643,7 @@ namespace Sequencer { this->broadcast.notice( function, "waiting for TCS operator to send \"ontarget\" signal" ); - // AUTO-ONTARGET (config TCS_AUTO_ONTARGET, default no): the TCS itself can + // AUTO-ONTARGET (config TCS_AUTO_ONTARGET, default yes): the TCS itself can // satisfy this wait through the same ontarget() the operator's press uses. // The watcher lives exactly as long as this wait (auto_ontarget_active). // @@ -2625,7 +2654,7 @@ namespace Sequencer { // database coordinates differ by the pointmode focal-plane offset. // ontarget_gen was taken, retiring older watchers, before the COORDS. std::thread( &Sequencer::Sequence::dothread_auto_ontarget, this, - ra_out, dec_out, ontarget_gen ).detach(); + ra_out, dec_out, angle_out, ontarget_gen ).detach(); } while ( !this->cancel_flag.load() && !this->is_ontarget.load() ) { diff --git a/sequencerd/sequence.h b/sequencerd/sequence.h index 9e6e7e13..67978046 100644 --- a/sequencerd/sequence.h +++ b/sequencerd/sequence.h @@ -402,7 +402,7 @@ namespace Sequencer { /** @brief satisfies the TCSOP wait when the TCS itself reports arrival * (config TCS_AUTO_ONTARGET); coords are the commanded SCOPE * frame position in degrees; see sequence.cpp */ - void dothread_auto_ontarget( double tgt_ra_deg, double tgt_dec_deg, unsigned gen ); + void dothread_auto_ontarget( double tgt_ra_deg, double tgt_dec_deg, double tgt_cass_deg, unsigned gen ); std::atomic auto_ontarget_active{false}; ///< true only while move_to_target's TCSOP wait is open std::atomic auto_ontarget_gen{0}; ///< launch generation; a watcher retires when a newer one is launched std::mutex ontarget_ack_mtx; ///< a watcher's acknowledgment vs a new move retiring watchers and clearing is_ontarget @@ -429,8 +429,9 @@ namespace Sequencer { double tcs_domeazi_ready; ///< max degrees azimuth that dome and telescope can differ before ready to observe double tcs_preauth_time; ///< seconds before end of exposure to notify TCS of next target's coords (0 to disable) double offset_settle_sec; ///< sec to wait after a target offset before exposing (config OFFSET_SETTLE_SEC; 0 disables) - bool tcs_auto_ontarget = false; ///< TCS ?ONTARGET may satisfy the TCSOP wait (config TCS_AUTO_ONTARGET; default no) + bool tcs_auto_ontarget = true; ///< TCS ?ONTARGET may satisfy the TCSOP wait (config TCS_AUTO_ONTARGET; default yes) double tcs_auto_ontarget_sep = 5.0; ///< max |telescope-target| arcsec for auto-ontarget (config TCS_AUTO_ONTARGET_SEP) + double tcs_auto_ontarget_cass = 0.05; ///< max |TCS cass angle - commanded ring angle| deg for auto-ontarget (config TCS_AUTO_ONTARGET_CASS) /// std::mutex tcs_ontarget_mtx; /// std::condition_variable tcs_ontarget_cv; diff --git a/sequencerd/sequencer_server.cpp b/sequencerd/sequencer_server.cpp index 56ea79d7..9b37a371 100644 --- a/sequencerd/sequencer_server.cpp +++ b/sequencerd/sequencer_server.cpp @@ -372,6 +372,21 @@ namespace Sequencer { applied++; } + // TCS_AUTO_ONTARGET_CASS + if (config.param[entry] == "TCS_AUTO_ONTARGET_CASS") { + try { + this->sequence.tcs_auto_ontarget_cass = std::stod( config.arg[entry] ); + } + catch (const std::exception &e) { + message.str(""); message << "ERROR parsing TCS_AUTO_ONTARGET_CASS: " << e.what(); + logwrite( function, message.str() ); + return ERROR; + } + message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; + logwrite( function, message.str() ); + applied++; + } + // TCS_AUTO_ONTARGET_SEP if (config.param[entry] == "TCS_AUTO_ONTARGET_SEP") { try { From f678031999d13bf016853edbf22f482ca6e350c3 Mon Sep 17 00:00:00 2001 From: Christoffer Fremling Date: Thu, 29 Oct 2026 16:07:10 +0200 Subject: [PATCH 08/10] camerad, sequencerd: stop and abort an NEXP>1 series stop: the sequencer accepts it for the whole series of the current target (the EXPOSE wait covers only the first sub-exposure; camerad not READY covers the rest) and no longer clears a cancel. camerad ends the series after the frame in progress, cut short if it is integrating with 5 s or more left, instead of starting the next sub-exposure. abort no longer starts the next sub-exposure either. A cancelled exposure's remaining PIXELCOUNT messages cannot set the READOUT wait later. camerad waits for the last frame's FITS write before acting on the next command after READY. Co-Authored-By: Claude Opus 5.5 --- camerad/astrocam.cpp | 44 +++++++++++++++++++++++++++++++---------- camerad/astrocam.h | 22 ++++++++++++++++++++- sequencerd/sequence.cpp | 18 +++++++++++++---- 3 files changed, 69 insertions(+), 15 deletions(-) diff --git a/camerad/astrocam.cpp b/camerad/astrocam.cpp index 6a5792d1..13a12dd8 100644 --- a/camerad/astrocam.cpp +++ b/camerad/astrocam.cpp @@ -957,6 +957,10 @@ namespace AstroCam { long Interface::do_abort() { std::string function = "AstroCam::Interface::do_abort"; std::stringstream message; + // No sub-exposure follows an abort: clearing exposure_pending below wakes + // dothread_monitor_exposure_pending, which starts the next sub-exposure + // whenever nexp is still > 0 after its decrement. + this->nexp.store(0); // int this_expbuf = this->get_expbuf(); for ( const auto &dev : this->active_devnums ) { this->exposure_pending( dev, false ); @@ -1019,9 +1023,10 @@ namespace AstroCam { return ERROR; } - // Don't make any changes while frames are still being written. + // Don't make any changes while frames are still being written: give a + // write that is finishing (ms after READY) up to 2 s, then refuse. // - if ( this->any_writes_pending() ) { + if ( !this->wait_writes_done( 2000 ) ) { logwrite( function, "ERROR: cannot change binning while frame write is in progress" ); retstring="write_in_progress"; return ERROR; @@ -2713,7 +2718,7 @@ namespace AstroCam { if ( interface.nexp.load() > 0 ) { message.str(""); message << "starting next exposure. " << interface.nexp << " remaining"; interface.broadcast.notice( function, message.str() ); - interface.do_expose(interface.nexp); + interface.do_expose(interface.nexp, true); // continuation: nexp already holds what is left } else { interface.status.can_expose.store(true); @@ -2768,7 +2773,7 @@ namespace AstroCam { * @return ERROR or NO_ERROR * */ - long Interface::do_expose(int nexp_in) { + long Interface::do_expose(int nexp_in, bool continuation) { const std::string function("AstroCam::Interface::do_expose"); std::stringstream message; std::string devstr; @@ -2792,8 +2797,6 @@ namespace AstroCam { } } - this->nexp.store(nexp_in); - // Different controllers may have different rules about when to set and clear their // exposure_pending flag (depending on whether or not they support frame transfer), but // if an exposure is pending for any controller then do not start a new exposure. @@ -2807,6 +2810,11 @@ namespace AstroCam { return(ERROR); } + // Only now, and only for a new expose command: a refused expose must not + // change how many sub-exposures the running series has left, and a + // continuation must not undo a stop that lowered nexp since it was read. + if ( !continuation ) this->nexp.store(nexp_in); + // When using the internal ARC shutter control you cannot overlap exposures // with readout because it would require sending a command which cannot // be handled during readout. @@ -3602,9 +3610,10 @@ for ( const auto &dev : selectdev ) { return( ERROR ); } - // Don't remap the DMA buffer while frames are still being written from it. + // Don't remap the DMA buffer while frames are still being written from it: + // give a write that is finishing (ms after READY) up to 2 s, then refuse. // - if ( this->any_writes_pending() ) { + if ( !this->wait_writes_done( 2000 ) ) { logwrite( function, "ERROR: cannot remap DMA buffer while frame write is in progress" ); retstring="write_in_progress"; return ERROR; @@ -4427,9 +4436,18 @@ logwrite(function, message.str()); const std::string function("AstroCam::Interface::stop_exposure"); std::stringstream message; - // nothing to do if not exposing - // + // An NEXP>1 series ends after the frame in progress, whatever that frame + // is doing -- integrating, reading out, or about to start -- so no further + // sub-exposure begins. That frame is also cut short below when it is + // integrating with >= 5 s left. + const bool ends_series = ( this->nexp.load() > 1 ); + if ( ends_series ) this->nexp.store(1); if ( ! this->exposure_pending() ) { + if ( ends_series ) { + logwrite( function, "between sub-exposures: the series ends after the next frame" ); + retstring="series_ends"; + return NO_ERROR; + } logwrite( function, "not exposing" ); return NOTHING; } @@ -4440,6 +4458,12 @@ logwrite(function, message.str()); // long remaining_time = this->camera.shutter_timer.get_remaining(); + if ( remaining_time < 5000 && ends_series ) { + logwrite( function, "too late to cut this frame short: the series ends after it" ); + retstring="series_ends"; + return NO_ERROR; + } + if ( remaining_time < 5000 ) { message << "ERROR cannot stop exposure time with less than 5 s exptime remaining" << " (" << remaining_time << ")"; diff --git a/camerad/astrocam.h b/camerad/astrocam.h index 0901b248..f225ab9a 100644 --- a/camerad/astrocam.h +++ b/camerad/astrocam.h @@ -964,6 +964,26 @@ std::vector> fitsinfo; /***** Interface::any_writes_pending ************************************/ + /***** Interface::wait_writes_done **************************************/ + /** + * @brief wait up to timeout_ms for every frame write to finish + * @return true if no writes are pending, false on timeout + * @details READY is published when the last frame has arrived, a few ms + * before its FITS write ends, so a command sent right after READY + * can find a write still in progress. + */ + inline bool wait_writes_done( int timeout_ms ) { + std::unique_lock lock( this->write_lock ); + return this->write_condition.wait_for( lock, std::chrono::milliseconds( timeout_ms ), [this]() { + for ( const auto &pending : this->writes_pending ) { + if ( !pending.empty() ) return false; + } + return true; + } ); + } + /***** Interface::wait_writes_done **************************************/ + + /***** Interface::write_pending *****************************************/ /** * @brief Set or clear the write pending state for a given exposure @@ -1240,7 +1260,7 @@ std::vector> fitsinfo; void set_imagesize(int rowsin, int colsin, int* status); long expose(std::string nexp_in); - long do_expose(int nexp_in); + long do_expose(int nexp_in, bool continuation=false); ///< continuation: the next sub-exposure of a running series double get_live_airmass(); ///< latest airmass from cached tcsd telemetry, or NAN void get_live_provenance( std::string &githash, std::string &buildtime ); ///< build provenance of all running daemons long native(std::string cmdstr); diff --git a/sequencerd/sequence.cpp b/sequencerd/sequence.cpp index aa24ba62..5aa8d9d3 100644 --- a/sequencerd/sequence.cpp +++ b/sequencerd/sequence.cpp @@ -1054,6 +1054,7 @@ namespace Sequencer { // if ( this->cancel_flag.load() ) { this->broadcast.notice( function, "exposure cancelled" ); + this->arm_readout_flag = false; // the stopped frame's PIXELCOUNTs must not set READOUT later error = this->target.update_state( Sequencer::TARGET_UNASSIGNED ); message.str(""); message << ( error==NO_ERROR ? "" : "ERROR " ) << "marking target " << this->target.name << " id " << this->target.obsid << " order " << this->target.obsorder @@ -3209,15 +3210,24 @@ namespace Sequencer { ScopedState thr_state( this->thread_state_manager, Sequencer::THR_STOP_EXPOSURE ); - // This function is only used while exposing + // Only while an exposure command of the current target is running. For + // NEXP>1 that is the whole series: the EXPOSE bit covers only the first + // sub-exposure; after it, camerad not READY (and not the previous + // target's last readout after a pick, which has nothing left to stop) + // covers the rest. A stale READOUT bit is not evidence. camerad ends the + // series after the frame in progress, cut short if it is integrating + // with >= 5 s left. A stop never clears a cancel. // - if ( ! this->wait_state_manager.is_set( Sequencer::SEQ_WAIT_EXPOSE ) ) { + bool series_running = false; + { + std::lock_guard lock( this->camerad_mtx ); + series_running = !this->can_expose.load() && !this->readout_gate_armed; + } + if ( ! this->wait_state_manager.is_set( Sequencer::SEQ_WAIT_EXPOSE ) && !series_running ) { this->broadcast.notice(function, "not currently exposing"); return; } - this->cancel_flag.store(false); - // Send command to the camera to stop the exposure. // std::string reply; From 8c251201c229a45fea85b97bcaa198964f313740 Mon Sep 17 00:00:00 2001 From: Christoffer Fremling Date: Thu, 29 Oct 2026 16:10:18 +0200 Subject: [PATCH 09/10] sequencerd: expose once acamd has made the target offset A target offset sent while ACAM guides only changes acamd's goal; the guide loop's next correction makes the move. The sequencer exposed a fixed OFFSET_SETTLE_SEC (3 s) after sending it, which could be before the telescope had moved. It now waits until acamd's status shows the goal change raised after the offset was sent and cleared again (GOALSHIFT_PENDING), bounded at 30 s, after which it exposes anyway with a warning; a cancel ends the wait. OFFSET_SETTLE_SEC is now an extra wait after the move and defaults to 0. Co-Authored-By: Claude Opus 5.5 --- Config/sequencerd.cfg.in | 2 +- sequencerd/sequence.cpp | 101 +++++++++++++++++++++++++++++++++++++-- sequencerd/sequence.h | 9 ++-- 3 files changed, 103 insertions(+), 9 deletions(-) diff --git a/Config/sequencerd.cfg.in b/Config/sequencerd.cfg.in index f0a5c079..b50de05b 100644 --- a/Config/sequencerd.cfg.in +++ b/Config/sequencerd.cfg.in @@ -162,7 +162,7 @@ ACQUIRE_TIMEOUT=90 # seconds before ACAM acquisition sequence ACQUIRE_RETRYS=5 # max number of retrys before acquisition fails (optional, can be left blank to disable) ACQUIRE_OFFSET_THRESHOLD=0.5 # computed offset below this threshold (in arcsec) defines successful acquisition ACQUIRE_MIN_REPEAT=2 # minimum number of sequential successful acquires -OFFSET_SETTLE_SEC=3 # sec to wait after an offset-star target offset before exposing (0 disables; only delays targets with a non-zero offset) +OFFSET_SETTLE_SEC=0 # extra sec to wait before exposing once acamd has made an offset-star target offset (the sequencer waits for the move itself; 0 = no extra wait) ACQUIRE_TCS_MAX_OFFSET=60 # the maximum allowable offset sent to the TCS, in arcsec # Calibration Settings diff --git a/sequencerd/sequence.cpp b/sequencerd/sequence.cpp index 5aa8d9d3..c42891c9 100644 --- a/sequencerd/sequence.cpp +++ b/sequencerd/sequence.cpp @@ -197,6 +197,13 @@ namespace Sequencer { int64_t pubtime=0; Common::extract_telemetry_value( jmessage, Key::PUBTIME, pubtime ); + // acamd's goal change still to be made: raised by offsetgoal, cleared once + // the correction computed against the new goal has been sent and the TCS + // has answered it. wait_for_target_offset waits on it before exposing. + const bool has_goalshift = jmessage.contains( Key::Acamd::GOALSHIFT_PENDING ); + bool goalshift = false; + if ( has_goalshift ) Common::extract_telemetry_value( jmessage, Key::Acamd::GOALSHIFT_PENDING, goalshift ); + // Store under the mutex so the writes are visible to any waiter's predicate // when it re-acquires acam_mtx inside acam_cv.wait(). std::lock_guard lock(this->acam_mtx); @@ -204,6 +211,12 @@ namespace Sequencer { if ( has_mode ) { this->is_acam_acquiring.store( acquiring ); } + // recorded per status, so a wait that starts after the change was already + // made still sees that it was raised and then cleared + if ( has_goalshift ) { + if ( goalshift ) this->acam_goalshift_raised_us.store( pubtime ); + else this->acam_goalshift_cleared_us.store( pubtime ); + } this->acam_pubtime.store( pubtime ); this->acam_cv.notify_all(); } @@ -944,12 +957,13 @@ namespace Sequencer { // re-applied during each readout (flexure_reapply, in the readout wait). // std::future flexure_future; + int64_t offset_sent_us = 0; // set when the target offset went to acamd as a goal change if ( !this->target.iscal ) { // send offsets only on fineacquire, otherwise user needs to fix things if ( !this->is_repeat_target.load() && this->is_fineacquire_locked.load() && - this->target_offset() == ERROR ) { + this->target_offset( &offset_sent_us ) == ERROR ) { if (this->wait_for_user()==ABORT) { this->broadcast.notice( function, "cancelled" ); return; @@ -968,9 +982,25 @@ namespace Sequencer { } } - // The offset-star offset is applied asynchronously by the ACAM guide loop; - // ACAMD_OFFSETGOAL returns before the move lands. Wait so the target is on - // the slit before exposing. Skipped when the target has no offset. + // A target offset sent while ACAM guides only changes acamd's goal. The + // guide loop makes the move with its next correction, whose pt_offset + // returns when the TCS answers the move. Expose once acamd reports that + // move made; the wait overlaps the slit move above and is bounded. If the + // move is not reported in time, expose anyway with a warning -- what the + // fixed settle always did -- so this can never hold the sequence. + if ( offset_sent_us > 0 ) { + const long offset_made = this->wait_for_target_offset( offset_sent_us ); + if ( offset_made == ABORT ) { + this->broadcast.notice( function, "cancelled" ); + return; + } + if ( offset_made != NO_ERROR ) { + this->broadcast.warning( function, "target offset not confirmed by acamd in time: exposing anyway" ); + } + } + + // Optional extra settle once the offset has been made (OFFSET_SETTLE_SEC, + // 0 = none). Skipped when the target has no offset. if ( !this->target.iscal && this->is_fineacquire_locked.load() && ( this->target.offset_ra != 0.0 || this->target.offset_dec != 0.0 ) && this->offset_settle_sec > 0.0 ) { @@ -4354,9 +4384,11 @@ namespace Sequencer { * @return ERROR or NO_ERROR * */ - long Sequence::target_offset() { + long Sequence::target_offset( int64_t *acam_sent_us ) { const std::string function("Sequencer::Sequence::target_offset"); + if ( acam_sent_us ) *acam_sent_us = 0; + // nothing to do if both ra and dec offsets are zero if (this->target.offset_ra == 0.0 && this->target.offset_dec == 0.0) return NO_ERROR; @@ -4371,7 +4403,11 @@ namespace Sequencer { const double ddec_deg = this->target.offset_dec / 3600.0; std::stringstream cmd; cmd << ACAMD_OFFSETGOAL << " " << std::fixed << std::setprecision(6) << dra_deg << " " << ddec_deg; + // taken before the send, so acamd's publish of the raised goal change is + // always later (no guard band needed, unlike do_acam_acquire) + const int64_t sent_us = get_time_us(); error = this->acamd.command( cmd.str() ); + if ( error == NO_ERROR && acam_sent_us ) *acam_sent_us = sent_us; } else // if ACAM is not guiding then I send the offsets directly to the TCS @@ -4390,6 +4426,61 @@ namespace Sequencer { /***** Sequencer::Sequence::target_offset ***********************************/ + /***** Sequencer::Sequence::wait_for_target_offset **************************/ + /** + * @brief wait until acamd has made a target offset sent as a goal change + * @details ACAMD_OFFSETGOAL only changes the guide goal. acamd's guide loop + * makes the move with its next correction, and that correction's + * pt_offset returns when the TCS answers the move (later for + * bigger moves). acamd raises GOALSHIFT_PENDING with the goal + * change and clears it once that correction has been sent and + * answered. The move is made once a status published after the + * send has shown it raised and a later one has shown it cleared. + * handletopic_acamd records both publish times as statuses + * arrive, so this holds even if the move was made before this + * wait began (the wait follows the slit move). Abort wakes acam_cv. + * @param[in] sent_us get_time_us() taken just before the offset was sent + * @return NO_ERROR | ERROR (not reported within TIMEOUT_SEC) | ABORT (cancelled) + * + */ + long Sequence::wait_for_target_offset( const int64_t sent_us ) { + const std::string function("Sequencer::Sequence::wait_for_target_offset"); + + // The move is made at acamd's next ACAM frame and answered by the TCS + // within seconds (measured: <= 4.3 s up to 147"); this covers a second + // frame and a 300" move. Bounded: the caller exposes either way. + // + constexpr double TIMEOUT_SEC = 30.0; + + logwrite( function, "NOTICE: waiting for acamd to make the target offset before exposing" ); + + const auto deadline = std::chrono::steady_clock::now() + std::chrono::duration( TIMEOUT_SEC ); + bool made = false; + { + std::unique_lock lock( this->acam_mtx ); + made = this->acam_cv.wait_until( lock, deadline, [&]() { + if ( this->cancel_flag.load() ) return true; + const int64_t raised = this->acam_goalshift_raised_us.load(); + return ( raised > sent_us && this->acam_goalshift_cleared_us.load() > raised ); + } ); + } + + if ( this->cancel_flag.load() ) return ABORT; + + std::ostringstream oss; + if ( !made ) { + oss << "ERROR acamd did not make the target offset within " << TIMEOUT_SEC << " s"; + logwrite( function, oss.str() ); + return ERROR; + } + oss << "target offset made by acamd " << std::fixed << std::setprecision(1) + << ( get_time_us() - sent_us ) / 1.0e6 << " s after it was sent"; + logwrite( function, oss.str() ); + return NO_ERROR; + } + /***** Sequencer::Sequence::wait_for_target_offset **************************/ + + /***** Sequencer::Sequence::publish_targetinfo *****************************/ /** * @brief publish target info on Topic::TARGETINFO, on change (or force) diff --git a/sequencerd/sequence.h b/sequencerd/sequence.h index 67978046..97328b4f 100644 --- a/sequencerd/sequence.h +++ b/sequencerd/sequence.h @@ -305,6 +305,8 @@ namespace Sequencer { std::atomic is_acam_guiding{false}; ///< is acam guiding (IS_ACQUIRED)? std::atomic is_acam_acquiring{false}; ///< is acam in an acquire mode? std::atomic acam_pubtime{0}; ///< publish time (us) of latest received acamd status + std::atomic acam_goalshift_raised_us{0}; ///< publish time (us) of the latest acamd status showing GOALSHIFT_PENDING set + std::atomic acam_goalshift_cleared_us{0}; ///< publish time (us) of the latest acamd status showing GOALSHIFT_PENDING clear /** @brief guard-band (us) subtracted from the acquire-command send time * when computing the freshness boundary in do_acam_acquire. Tolerates @@ -350,7 +352,7 @@ namespace Sequencer { tcs_settle_stable(1), tcs_domeazi_ready(1), tcs_preauth_time(0), - offset_settle_sec(3), + offset_settle_sec(0), do_once(false), tcs_which("real"), tcs_name("offline"), @@ -428,7 +430,7 @@ namespace Sequencer { double tcs_settle_stable; ///< time that TCS must report TRACKING before it is really tracking double tcs_domeazi_ready; ///< max degrees azimuth that dome and telescope can differ before ready to observe double tcs_preauth_time; ///< seconds before end of exposure to notify TCS of next target's coords (0 to disable) - double offset_settle_sec; ///< sec to wait after a target offset before exposing (config OFFSET_SETTLE_SEC; 0 disables) + double offset_settle_sec; ///< extra sec to wait after the target offset has been made, before exposing (config OFFSET_SETTLE_SEC; 0 = none) bool tcs_auto_ontarget = true; ///< TCS ?ONTARGET may satisfy the TCSOP wait (config TCS_AUTO_ONTARGET; default yes) double tcs_auto_ontarget_sep = 5.0; ///< max |telescope-target| arcsec for auto-ontarget (config TCS_AUTO_ONTARGET_SEP) double tcs_auto_ontarget_cass = 0.05; ///< max |TCS cass angle - commanded ring angle| deg for auto-ontarget (config TCS_AUTO_ONTARGET_CASS) @@ -621,7 +623,8 @@ namespace Sequencer { long get_tcs_weather_coords( double &ra_h, double &dec_d ); ///< read the current TCS ra,dec in decimal hr,deg long get_tcs_coords( double &ra_h, double &dec_d ); ///< read the current TCS ra,dec,cass in decimal hr,deg long get_tcs_cass( double &cass ); - long target_offset(); + long target_offset( int64_t *acam_sent_us = nullptr ); ///< acam_sent_us: send time if the offset went to acamd as a goal change, else 0 + long wait_for_target_offset( const int64_t sent_us ); ///< wait until acamd has made that goal change void publish_targetinfo( bool force=false ); ///< publish target info on change (or force) From e3414e7c651358ab7b2fe18d433a8d5342b6e71a Mon Sep 17 00:00:00 2001 From: Christoffer Fremling Date: Thu, 29 Oct 2026 16:53:34 +0200 Subject: [PATCH 10/10] slicecamd: do not miss a short-lived acamd goal-shift state The wait for acamd to apply a fine-acquire correction completed only if it saw GOALSHIFT_PENDING set and then clear. Condition-variable notifications do not queue, so if both statuses were handled before the waiting thread re-checked, it waited out FINE_ACQUIRE_MOVE_TIMEOUT and, under a correction budget, failed the run. The acamd telemetry handler now records the publish times of the latest set and clear statuses, and the wait completes once a set status published after the send is followed by a clear one, as the sequencer's wait for the target offset does. Co-Authored-By: Claude Opus 5.5 --- slicecamd/slicecam_interface.cpp | 27 +++++++++++++-------------- slicecamd/slicecam_interface.h | 3 ++- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/slicecamd/slicecam_interface.cpp b/slicecamd/slicecam_interface.cpp index edc2420f..0618c7e1 100644 --- a/slicecamd/slicecam_interface.cpp +++ b/slicecamd/slicecam_interface.cpp @@ -643,21 +643,16 @@ namespace Slicecam { // acamd reports the goal change applied before settling if ( was_guiding && this->fineacquire_state.move_timeout_sec > 0.0 ) { std::unique_lock lock(this->acam_mtx); - // completion requires seeing THIS request registered (pending true on a - // status after the send) and then cleared; statuses arrive in publish - // order, so the false cannot predate the request - bool seen_pending = false; + // completion requires a status after the send showing the request + // pending, followed by one showing it cleared const bool applied = this->acam_cv.wait_for( lock, std::chrono::duration( this->fineacquire_state.move_timeout_sec ), - [this, send_time, &seen_pending]() { + [this, send_time]() { if ( !this->is_fineacquire_running.load(std::memory_order_acquire) || !this->should_framegrab_run.load(std::memory_order_acquire) ) return true; - if ( this->last_acam_pubtime.load(std::memory_order_acquire) <= send_time ) return false; - if ( this->acam_goalshift_pending.load(std::memory_order_acquire) ) { - seen_pending = true; - return false; - } - return seen_pending; + const int64_t raised = this->acam_goalshift_raised_us.load(std::memory_order_acquire); + return ( raised > send_time && + this->acam_goalshift_cleared_us.load(std::memory_order_acquire) > raised ); }); if ( !this->should_framegrab_run.load(std::memory_order_acquire) ) return; if ( !this->is_fineacquire_running.load(std::memory_order_acquire) ) return; @@ -1082,9 +1077,13 @@ namespace Slicecam { Common::extract_telemetry_value( jmessage, Key::PUBTIME, pubtime ); this->last_acam_pubtime.store( pubtime, std::memory_order_relaxed ); - bool goalshift_pending=false; - Common::extract_telemetry_value( jmessage, Key::Acamd::GOALSHIFT_PENDING, goalshift_pending ); - this->acam_goalshift_pending.store( goalshift_pending, std::memory_order_relaxed ); + // when acamd last reported a goal change pending, and when it last reported none + if ( jmessage.contains( Key::Acamd::GOALSHIFT_PENDING ) ) { + bool goalshift_pending=false; + Common::extract_telemetry_value( jmessage, Key::Acamd::GOALSHIFT_PENDING, goalshift_pending ); + if ( goalshift_pending ) this->acam_goalshift_raised_us.store( pubtime, std::memory_order_relaxed ); + else this->acam_goalshift_cleared_us.store( pubtime, std::memory_order_relaxed ); + } // wake any thread waiting on ACAM state (e.g. fineacquire) std::lock_guard lock(this->acam_mtx); diff --git a/slicecamd/slicecam_interface.h b/slicecamd/slicecam_interface.h index 3a5eb9eb..922bdbcf 100644 --- a/slicecamd/slicecam_interface.h +++ b/slicecamd/slicecam_interface.h @@ -197,7 +197,8 @@ namespace Slicecam { std::atomic is_acam_guiding; ///< is acam guiding? std::atomic last_acam_pubtime{0}; ///< pubtime (us) of latest received acamd status - std::atomic acam_goalshift_pending{false}; ///< acamd has an unexecuted goal change + std::atomic acam_goalshift_raised_us{0}; ///< pubtime (us) of the latest acamd status showing GOALSHIFT_PENDING set + std::atomic acam_goalshift_cleared_us{0}; ///< pubtime (us) of the latest acamd status showing GOALSHIFT_PENDING clear // Latest target (goal) coords published on Topic::TARGETINFO // NAN until a TARGETINFO arrives, so manual runs with no sequencer target log nan.