diff --git a/Config/acamd.cfg.in b/Config/acamd.cfg.in index 3b101dee..8153adf1 100644 --- a/Config/acamd.cfg.in +++ b/Config/acamd.cfg.in @@ -157,10 +157,10 @@ MOTOR_POS="cover 1 1 504 close" # ACQUIRE_TIMEOUT=90 # seconds before ACAM acquisition sequence aborts on failure to acquire (REQUIRED!) ACQUIRE_RETRYS=5 # max number of retrys before acquisition fails (optional, can be left blank to disable) -ACQUIRE_OFFSET_THRESHOLD=0.5 # computed offset below this threshold (in arcsec) defines successful acquisition +ACQUIRE_OFFSET_THRESHOLD=1.0 # computed offset below this threshold (in arcsec) defines successful acquisition ACQUIRE_MIN_REPEAT=2 # minimum number of sequential successful acquires ACQUIRE_TCS_MAX_OFFSET=60 # max offset (arcsec) for an ordinary guiding correction sent to the TCS -ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET=300 # max offset (arcsec) for a deliberate goal offset (put-on-slit, offset-star, end-of-fineacquire, pyGUI 'Offset') applied while guiding +ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET=300 # max offset (arcsec) for the one correction that makes a deliberate goal offset (put-on-slit, offset-star, end-of-fineacquire, pyGUI 'Offset'); guide and fine-acquire corrections keep ACQUIRE_TCS_MAX_OFFSET # SkySimulator options: # SKYSIM_IMAGE_SIZE= where is integer diff --git a/Config/sequencerd.cfg.in b/Config/sequencerd.cfg.in index f0a5c079..b50de05b 100644 --- a/Config/sequencerd.cfg.in +++ b/Config/sequencerd.cfg.in @@ -162,7 +162,7 @@ ACQUIRE_TIMEOUT=90 # seconds before ACAM acquisition sequence ACQUIRE_RETRYS=5 # max number of retrys before acquisition fails (optional, can be left blank to disable) ACQUIRE_OFFSET_THRESHOLD=0.5 # computed offset below this threshold (in arcsec) defines successful acquisition ACQUIRE_MIN_REPEAT=2 # minimum number of sequential successful acquires -OFFSET_SETTLE_SEC=3 # sec to wait after an offset-star target offset before exposing (0 disables; only delays targets with a non-zero offset) +OFFSET_SETTLE_SEC=0 # extra sec to wait before exposing once acamd has made an offset-star target offset (the sequencer waits for the move itself; 0 = no extra wait) ACQUIRE_TCS_MAX_OFFSET=60 # the maximum allowable offset sent to the TCS, in arcsec # Calibration Settings diff --git a/Config/slicecamd.cfg.in b/Config/slicecamd.cfg.in index 3bd6316a..c481b4a3 100644 --- a/Config/slicecamd.cfg.in +++ b/Config/slicecamd.cfg.in @@ -86,9 +86,10 @@ FINE_ACQUIRE_MIN_SAMPLES=3 # FINE_ACQUIRE_SETTLE_FRAMES= # frames discarded after each telescope move to allow settling before -# centroid measurements resume. +# centroid measurements resume. The wall-clock settle (FINE_ACQUIRE_SETTLE_SEC, +# applied after acamd confirms the goal change) covers the move itself. # -FINE_ACQUIRE_SETTLE_FRAMES=2 +FINE_ACQUIRE_SETTLE_FRAMES=0 # FINE_ACQUIRE_SETTLE_SEC= # Time-based settle applied after each move, in addition to SETTLE_FRAMES, @@ -98,6 +99,13 @@ FINE_ACQUIRE_SETTLE_FRAMES=2 # FINE_ACQUIRE_SETTLE_SEC=3.0 +# FINE_ACQUIRE_MOVE_TIMEOUT= +# Max time to wait for acamd to report a requested goal change applied +# before the settle begins. On timeout the settle proceeds anyway. Set to 0 +# to disable the wait. +# +FINE_ACQUIRE_MOVE_TIMEOUT=60.0 + # FINE_ACQUIRE_GAIN= # Proportional gain = {0..1} applied to the commanded offset when the residual # is at or below FINE_ACQUIRE_GAIN_THRESHOLD arcsec. diff --git a/acamd/acam_interface.cpp b/acamd/acam_interface.cpp index f3a0daf1..7e0b47c9 100644 --- a/acamd/acam_interface.cpp +++ b/acamd/acam_interface.cpp @@ -1443,6 +1443,7 @@ namespace Acam { const int attempts = this->target.attempts; const std::string filter = this->motion.get_current_filtername(); const std::string cover = this->motion.get_current_coverpos(); + const bool goalshift_pending = this->target.goalshift_pending.load(); // unless forced, only publish if there was a change in any one of these // @@ -1452,7 +1453,8 @@ namespace Acam { nacquired == this->last_status.nacquired && attempts == this->last_status.attempts && filter == this->last_status.filter && - cover == this->last_status.cover ) return; + cover == this->last_status.cover && + goalshift_pending == this->last_status.goalshift_pending ) return; this->last_status.acquire_mode = acquire_mode; this->last_status.is_acquired = is_acquired; @@ -1460,6 +1462,7 @@ namespace Acam { this->last_status.attempts = attempts; this->last_status.filter = filter; this->last_status.cover = cover; + this->last_status.goalshift_pending = goalshift_pending; // assemble the telemetry into a json message // @@ -1473,6 +1476,7 @@ namespace Acam { jmessage_out[Key::Acamd::BACKGROUND] = this->astrometry.get_background(); jmessage_out[Key::Acamd::FILTER] = filter; jmessage_out[Key::Acamd::COVER] = cover; + jmessage_out[Key::Acamd::GOALSHIFT_PENDING] = goalshift_pending; jmessage_out[Key::PUBTIME] = get_time_us(); try { @@ -3221,13 +3225,15 @@ logwrite( function, "[DEBUG] acquire here" ); } this->astrometry.solve( last_imagename, this->target.ext_solver_args ); - this->astrometry.get_solution( result, this->target.acam_goal.ra, this->target.acam_goal.dec, this->target.acam_goal.angle, matches, rmsarcsec ); + double solved_ra=NAN, solved_dec=NAN, solved_angle=NAN; + this->astrometry.get_solution( result, solved_ra, solved_dec, solved_angle, matches, rmsarcsec ); + this->target.set_goal( solved_ra, solved_dec, solved_angle ); // under goal_mtx: the guide loop may be reading it if ( result=="NOISY" || result=="GOOD" ) { goodsolution=true; - message.str(""); message << "setting goals from solver: " << this->target.acam_goal.ra << " " - << this->target.acam_goal.dec << " " - << this->target.acam_goal.angle; + message.str(""); message << "setting goals from solver: " << solved_ra << " " + << solved_dec << " " + << solved_angle; logwrite( function, message.str() ); break; } @@ -3307,11 +3313,13 @@ logwrite( function, message.str() ); message.str(""); message << "[DEBUG] input coords_slit.ra="<< this->target.coords_slit.ra << " .dec=" << this->target.coords_slit.dec; logwrite( function, message.str() ); + double goal_ra=NAN, goal_dec=NAN, goal_angle=NAN; this->fpoffsets.compute_offset_last_angle( this->target.get_pointmode(), "ACAM", this->target.coords_slit.ra, this->target.coords_slit.dec, - this->target.acam_goal.ra, this->target.acam_goal.dec, this->target.acam_goal.angle ); + goal_ra, goal_dec, goal_angle ); + this->target.set_goal( goal_ra, goal_dec, goal_angle ); // under goal_mtx: the guide loop may be reading it -message.str(""); message << "[DEBUG] goals: " << this->target.acam_goal.ra << " " << this->target.acam_goal.dec << " " << this->target.acam_goal.angle; +message.str(""); message << "[DEBUG] goals: " << goal_ra << " " << goal_dec << " " << goal_angle; logwrite( function, message.str() ); if ( error == NO_ERROR ) { @@ -3392,7 +3400,10 @@ logwrite( function, message.str() ); this->nacquired = 0; this->attempts = 0; this->sequential_failures = 0; - this->allow_large_offset.store(false); // no stale deliberate-offset allowance + { + std::lock_guard lock(this->goal_mtx); + this->deliberate_gen = 0; // no stale deliberate-offset allowance + } this->is_acquired.store( false, std::memory_order_release ); // Start the timeout clock, initialized as the time now plus the @@ -3466,6 +3477,7 @@ logwrite( function, message.str() ); // an attempt and will keep trying until max attempts. // long error = NO_ERROR; + bool frame_superseded = false; // the goal changed while this frame's correction was computed if ( this->acquire_mode == Acam::TARGET_ACQUIRE || this->acquire_mode == Acam::TARGET_ACQUIRE_HERE || @@ -3521,8 +3533,11 @@ message.str(""); message << "[DEBUG] ACQUIRE_HERE using goals: " << this->acam_g logwrite( function, message.str() ); } ******/ -message.str(""); message << "[DEBUG] using goals: " << this->acam_goal.ra << " " << this->acam_goal.dec << " " << this->acam_goal.angle; +{ +const goal_snapshot_t logged = this->snapshot_goal(); +message.str(""); message << "[DEBUG] using goals: " << logged.ra << " " << logged.dec << " " << logged.angle; logwrite( function, message.str() ); +} message.str(""); message << "[DEBUG] dRA=" << iface->target.dRA << " dDEC=" << iface->target.dDEC; logwrite( function, message.str() ); @@ -3586,7 +3601,8 @@ logwrite( function, message.str() ); // // Offsets calculated as difference between acam goals and the solution from // solve. ACAM goals are the SLIT->ACAM translated coords from the DB and - // are where the ACAM needs to be pointed. Goals never change. + // are where the ACAM needs to be pointed. Goals change only through + // offset_goal and a new acquisition (set_goal); see goal_mtx. // double ra_off, dec_off; // calculated offsets will be in degrees @@ -3609,13 +3625,31 @@ logwrite( function, message.str() ); offset = angular_separation( acam_goal.ra, acam_goal.dec, acam_ra, acam_dec ); } *****/ + // Every correction is classified by the goal it is computed against, read + // together with that goal's generation (PAL-171; see goal_mtx): + // + // - guide correction: the loop has already moved to this goal. Capped at + // ACQUIRE_TCS_MAX_OFFSET and, while guiding, median-filtered. It makes + // no goal change and spends nothing that belongs to one. + // - goal-change correction: offset_goal changed the goal and no correction + // computed against it has been sent. A fine-acquire change + // ('fineguiding') keeps the ACQUIRE_TCS_MAX_OFFSET cap; a deliberate + // change among those still to be made allows + // ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET. Sent at once and alone, never + // through the median filter. Once sent it has made the change(s) and + // clears goalshift_pending, unless a newer change arrived meanwhile. + // + // So a guide correction in flight when a deliberate offset arrives can + // neither use nor spend the deliberate cap. + // + const goal_snapshot_t goal = this->snapshot_goal(); if ( iface->fpoffsets.solve_offset( acam_ra, acam_dec, - this->acam_goal.ra, this->acam_goal.dec, + goal.ra, goal.dec, ra_off, dec_off ) == ERROR ) break; - offset = angular_separation( this->acam_goal.ra, this->acam_goal.dec, acam_ra, acam_dec ); + offset = angular_separation( goal.ra, goal.dec, acam_ra, acam_dec ); message.str(""); message << "[DEBUG] acam_ra=" << acam_ra << " acam_dec=" << acam_dec << " acam_goal.ra=" - << acam_goal.ra << " .dec=" << this->acam_goal.dec << " .ang=" << this->acam_goal.angle; + << goal.ra << " .dec=" << goal.dec << " .ang=" << goal.angle; logwrite( function, message.str() ); message.str(""); message << "[DEBUG] calculated RAoffset=" << ra_off << " DECLoffset=" << dec_off; logwrite( function, message.str() ); @@ -3634,18 +3668,18 @@ logwrite( function, message.str() ); // There is a maximum offset we send to the TCS. This is not a TCS limit // (theirs is very large); it is our safety limit so that a bad solution - // can't move us far off the slit. Ordinary guiding corrections use the - // normal tcs_max_offset (ACQUIRE_TCS_MAX_OFFSET). A deliberate goal offset - // applied while guiding -- via offset_goal, which covers put-on-slit, - // offset-star acquisition, the end-of-fineacquire target offset, and the - // pyGUI 'Offset' button -- is intentionally larger, so for the one - // correction that consumes it we allow up to tcs_max_putonslit_offset - // (ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET). The ACQUIRE path uses tcs_max_offset - // and is unaffected. + // can't move us far off the slit. Guide and fine-acquire corrections use + // tcs_max_offset (ACQUIRE_TCS_MAX_OFFSET). Only the correction that makes + // a deliberate goal offset -- put-on-slit, offset-star acquisition, the + // end-of-fineacquire target offset, the pyGUI 'Offset' button -- may move + // up to tcs_max_putonslit_offset (ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET). // - double maxoffset = this->tcs_max_offset; - if ( this->acquire_mode == Acam::TARGET_GUIDE && this->allow_large_offset.load() ) { - maxoffset = this->tcs_max_putonslit_offset; + const double maxoffset = goal.deliberate ? this->tcs_max_putonslit_offset : this->tcs_max_offset; + + if ( goal.goal_change ) { + message.str(""); message << "NOTICE correction for goal change " << goal.gen << ": offset=" << offset + << " arcsec, cap " << maxoffset << " arcsec"; + logwrite( function, message.str() ); } // Check the requested offset against the applicable max allowed offset @@ -3654,13 +3688,14 @@ logwrite( function, message.str() ); message.str(""); message << "[WARNING] calculated offset " << offset << " not below max " << maxoffset << " and will not be sent to the TCS"; logwrite( function, message.str() ); - this->allow_large_offset.store(false); // deliberate-offset allowance consumed even when rejected + // A refused correction has made no goal change: a deliberate change + // keeps its cap for the next frame. // Match found but failure to send an offset is considered an attempt // so attempts is incremented. // if ( attempts >= this->max_attempts ) { - message.str(""); message << "ERROR: failed to find offset below " << this->tcs_max_offset << " within max number of attempts"; + message.str(""); message << "ERROR: failed to find offset below " << maxoffset << " within max number of attempts"; iface->async.enqueue_and_log( function, message.str() ); error = ERROR; break; @@ -3678,7 +3713,29 @@ logwrite( function, message.str() ); bool should_offset=false; - // While guiding, run the offsets through a median filter over a + // A correction computed against a goal that has changed since is + // neither sent nor filtered: the next frame is computed against the new + // goal. + // + if ( !this->goal_unchanged_since( goal.gen ) ) { + // Measured against a goal that no longer exists: not sent, and the + // frame counts as neither an acquisition nor a failure. + logwrite( function, "NOTICE goal changed while this correction was computed: not sent" ); + attempts = 0; + frame_superseded = true; + break; + } + else + // A goal-change correction is sent at once and alone. It never enters + // the median filter, and the samples there, computed against the old + // goal, are discarded. + // + if ( goal.goal_change ) { + this->reset_offset_params(); + should_offset = true; + } + else + // While guiding, run guide corrections through a median filter over a // specified time period, and send the offsets only at the rate // determined by that period. This returns true and modifies the // values ra_off,dec_off when it's time to offset. @@ -3691,7 +3748,10 @@ logwrite( function, message.str() ); if ( should_offset ) { // send offset to TCS here (returns when offset is complete) if ( iface->tcsd.pt_offset( ra_off*3600., dec_off*3600., OFFSETRATE )==ERROR) break; - this->allow_large_offset.store(false); // deliberate-offset allowance consumed + // A goal-change correction has now made its change(s). A guide + // correction makes none: a goal change that arrived while it was in + // flight keeps its own cap and stays pending. + if ( goal.goal_change ) this->goal_change_made( goal.gen ); std::this_thread::sleep_for( std::chrono::seconds(1) ); } @@ -3727,7 +3787,7 @@ logwrite( function, message.str() ); } while ( false ); // the do-loop is executed only once. used to allow breaks. } // end of acquisition sequence - if ( this->nacquired == 0 && this->max_attempts > 0 ) { + if ( !frame_superseded && this->nacquired == 0 && this->max_attempts > 0 ) { logwrite(function,"[DEBUG] nacquired="+std::to_string(nacquired)); if ( ++this->sequential_failures >= this->max_attempts ) { logwrite( function, "ERROR sequential failures "+std::to_string(this->sequential_failures) @@ -5474,8 +5534,9 @@ logwrite( function, message.str() ); /***** Acam::Interface::offset_goal *****************************************/ /** * @brief sets the dRA,dDEC offsets to apply to goal - * @details The offset is not applied here, this only stores the offsets - * in the class. Offsets are applied only while guiding. + * @details This changes the goal only. The acquisition or guide loop + * moves the telescope with its next correction computed against + * the new goal (see goal_mtx in Target). * @param[in] args input string expected " " * @param[out] retstring return string * @return ERROR | NO_ERROR @@ -5497,7 +5558,7 @@ logwrite( function, message.str() ); retstring = ACAMD_OFFSETGOAL; retstring.append( " [ [ fineguiding ] ]\n" ); retstring.append( " Apply offsets to the ACAM goal coordinates.\n" ); - retstring.append( " These offsets are applied only while guiding. If omitted,\n" ); + retstring.append( " The acquisition or guide loop then moves to the new goal. If omitted,\n" ); retstring.append( " the current offsets are returned. Units are in degrees.\n" ); retstring.append( " 'fineguiding' marks a slicecam fine-guiding correction (keeps the 60\" cap).\n" ); return HELP; @@ -5512,27 +5573,48 @@ logwrite( function, message.str() ); retstring="invalid_argument"; return ERROR; } - this->target.dRA = dRA; - this->target.dDEC = dDEC; - - // optional fineguiding flag: marks a slicecam fine-guiding correction, which + // optional fineguiding flag: marks a slicecam fine-acquire correction, which // must stay within the ordinary 60" guiding cap. Its absence means a // deliberate goal offset (put-on-slit, offset-star, end-of-fineacquire, // pyGUI 'Offset') that may legitimately be larger. std::string flag; const bool is_fineguiding = (iss >> flag && flag == "fineguiding"); - // Apply any dRA, dDEC goal offsets from the "put on slit" action to - // acam_ra_goal, acam_dec_goal. These dRA,dDEC offsets can come from - // either the ACAM or slicecam GUIs and are stored in the Target class. + // Apply the dRA,dDEC offsets to the goal and record the change with its + // kind, in one step under goal_mtx: the guide loop never reads half a + // change, and two offsets arriving on different connections are both kept. + // Only a correction the guide loop computes against this goal (or a later + // one) can make the change; for a deliberate offset that correction alone + // may move up to ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET, whatever mode the loop is + // in. A correction already in flight can neither use nor spend it (PAL-171). // - // The supplied acam_ra_goal,acam_dec_goal are modified by dRA,dDEC - // - this->fpoffsets.apply_offset( this->target.acam_goal.ra, this->target.dRA, - this->target.acam_goal.dec, this->target.dDEC ); + unsigned long long gen=0; + { + std::lock_guard lock( this->target.goal_mtx ); + double goal_ra = this->target.acam_goal.ra; + double goal_dec = this->target.acam_goal.dec; + if ( this->fpoffsets.apply_offset( goal_ra, dRA, goal_dec, dDEC ) == ERROR ) { + logwrite( function, "ERROR applying the offset to the goal: goal unchanged" ); + retstring="goal_unchanged"; + return ERROR; + } + this->target.acam_goal.ra = goal_ra; + this->target.acam_goal.dec = goal_dec; + this->target.dRA = dRA; + this->target.dDEC = dDEC; + gen = ++this->target.goal_gen; + if ( !is_fineguiding && ( dRA != 0.0 || dDEC != 0.0 ) ) { + this->target.deliberate_gen = gen; // a zero offset changes nothing, so it raises no cap + } + this->target.goalshift_pending.store( true, std::memory_order_release ); + } + this->publish_status(true); + message.str(""); message << "goal change " << gen << " (" << ( is_fineguiding ? "fine-acquire" : "deliberate offset" ) + << "): dRA=" << dRA*3600. << " dDEC=" << dDEC*3600. << " arcsec"; + logwrite( function, message.str() ); - message.str(""); message << this->target.dRA << " " << this->target.dDEC; + message.str(""); message << dRA << " " << dDEC; retstring = message.str(); // Applying an offset to the goal while guiding must never drop out of @@ -5540,14 +5622,8 @@ logwrite( function, message.str() ); // TARGET_GUIDE and reset the offset filter so the new goal takes effect // quickly. This covers GUI "put on slit" and sequencer target offsets. // - // This is a deliberate offset (put-on-slit, offset-star, end-of-fineacquire, - // or the pyGUI 'Offset' button) and may exceed the normal guiding cap, so - // allow the next correction up to ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET. The - // allowance is one-shot: do_acquire consumes it when the offset is sent. - // if ( this->target.acquire_mode == Acam::TARGET_GUIDE ) { this->target.reset_offset_params(); - if ( !is_fineguiding ) this->target.allow_large_offset.store(true); } this->publish_status(); diff --git a/acamd/acam_interface.h b/acamd/acam_interface.h index 4c2f59fd..4648bffa 100644 --- a/acamd/acam_interface.h +++ b/acamd/acam_interface.h @@ -349,9 +349,10 @@ namespace Acam { std::atomic is_acquired; ///< set if target acquired successfully std::atomic stop_acquisition; ///< set if the acquisition sequence should stop + std::atomic goalshift_pending{false}; ///< a requested goal change awaits execution by the guide loop double tcs_max_offset; - double tcs_max_putonslit_offset{300.}; ///< max offset (arcsec) for a deliberate goal offset (put-on-slit etc.) applied while guiding; defaults 300 if ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET absent + double tcs_max_putonslit_offset{300.}; ///< max offset (arcsec) for the correction that makes a deliberate goal offset (put-on-slit etc.); defaults 300 if ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET absent double offset_cal_offset, offset_cal_raoff, offset_cal_decoff; @@ -386,7 +387,14 @@ namespace Acam { this->time_offs.clear(); this->tcs_offset_period = val; } - inline void reset_offset_params() { this->reset_offset_params(this->tcs_offset_period); } + /// clear the filter only: never rewrite tcs_offset_period, which a concurrent + /// offsetperiod command may be setting + inline void reset_offset_params() { + std::lock_guard lock(this->offset_params_mtx); + this->ra_offs.clear(); + this->dec_offs.clear(); + this->time_offs.clear(); + } inline double get_timeout() { return this->timeout; } @@ -483,9 +491,72 @@ namespace Acam { double ra; double dec; double angle; - } acam_goal; + } acam_goal; ///< guarded by goal_mtx (snapshot_goal, set_goal, offset_goal) - std::atomic allow_large_offset{false}; ///< one-shot: allow the next guiding correction up to tcs_max_putonslit_offset + // Goal changes and the corrections that make them (PAL-171). + // + // offset_goal changes the goal; the guide loop moves the telescope. Every + // correction is classified by the goal it was computed against: + // guide correction -- a goal the loop has already moved to; + // goal-change correction -- a goal changed since: a fine-acquire + // correction ('fineguiding') or a deliberate + // offset (untagged: target offset, offset star, + // put-on-slit, pyGUI 'Offset'). + // goal_gen counts goal changes (offset_goal) and replacements (set_goal, + // which counts as made). goal_gen_applied is the newest change that a SENT + // correction was computed against. deliberate_gen is the newest + // deliberate change (0 = none); it is still to be made while + // deliberate_gen > goal_gen_applied, and only a correction computed against + // it (or a later goal) can make it -- so a correction already in flight can + // neither use nor spend its cap. All guarded by goal_mtx. + // + std::mutex goal_mtx; + unsigned long long goal_gen{0}; + unsigned long long goal_gen_applied{0}; + unsigned long long deliberate_gen{0}; + + struct goal_snapshot_t { + double ra, dec, angle; + unsigned long long gen; ///< goal generation the correction is computed against + bool goal_change; ///< includes a goal change not yet made + bool deliberate; ///< includes a deliberate goal change not yet made + }; + + /// the goal, its generation and its classification, read together + inline goal_snapshot_t snapshot_goal() { + std::lock_guard lock(this->goal_mtx); + return { this->acam_goal.ra, this->acam_goal.dec, this->acam_goal.angle, this->goal_gen, + this->goal_gen > this->goal_gen_applied, + this->deliberate_gen > this->goal_gen_applied }; + } + + /// true if no goal change has arrived since the snapshot of generation gen + inline bool goal_unchanged_since( unsigned long long gen ) { + std::lock_guard lock(this->goal_mtx); + return this->goal_gen == gen; + } + + /// a correction computed against generation gen has been sent: the goal + /// changes up to gen are made. goalshift_pending stays set if a newer one arrived. + inline void goal_change_made( unsigned long long gen ) { + std::lock_guard lock(this->goal_mtx); + if ( gen > this->goal_gen_applied ) this->goal_gen_applied = gen; + this->goalshift_pending.store( this->goal_gen > this->goal_gen_applied, std::memory_order_release ); + } + + /// a new acquisition replaces the goal (from the target coordinates or a + /// solve): every earlier change is void and nothing is pending. The + /// generation moves on, so a correction computed against the replaced goal + /// is dropped; the new goal counts as made (the acquisition moves to it). + inline void set_goal( const double ra, const double dec, const double angle ) { + std::lock_guard lock(this->goal_mtx); + this->acam_goal.ra = ra; + this->acam_goal.dec = dec; + this->acam_goal.angle = angle; + this->goal_gen_applied = ++this->goal_gen; + this->deliberate_gen = 0; + this->goalshift_pending.store( false, std::memory_order_release ); + } Target() : iface(nullptr), timeout(10), max_attempts(-1), min_repeat(1), is_acquired(false), @@ -530,6 +601,7 @@ namespace Acam { int attempts = 0; std::string filter = ""; std::string cover = ""; + bool goalshift_pending = false; } last_status; public: diff --git a/common/message_keys.h b/common/message_keys.h index 0888bc52..de4e029b 100644 --- a/common/message_keys.h +++ b/common/message_keys.h @@ -106,6 +106,7 @@ namespace Key { inline const std::string ATTEMPTS = "attempts"; inline const std::string SEEING = "seeing"; inline const std::string BACKGROUND = "background"; + inline const std::string GOALSHIFT_PENDING = "goalshift_pending"; ///< a requested goal change awaits execution by the guide loop } namespace Slicecamd { diff --git a/sequencerd/sequence.cpp b/sequencerd/sequence.cpp index 5aa8d9d3..c42891c9 100644 --- a/sequencerd/sequence.cpp +++ b/sequencerd/sequence.cpp @@ -197,6 +197,13 @@ namespace Sequencer { int64_t pubtime=0; Common::extract_telemetry_value( jmessage, Key::PUBTIME, pubtime ); + // acamd's goal change still to be made: raised by offsetgoal, cleared once + // the correction computed against the new goal has been sent and the TCS + // has answered it. wait_for_target_offset waits on it before exposing. + const bool has_goalshift = jmessage.contains( Key::Acamd::GOALSHIFT_PENDING ); + bool goalshift = false; + if ( has_goalshift ) Common::extract_telemetry_value( jmessage, Key::Acamd::GOALSHIFT_PENDING, goalshift ); + // Store under the mutex so the writes are visible to any waiter's predicate // when it re-acquires acam_mtx inside acam_cv.wait(). std::lock_guard lock(this->acam_mtx); @@ -204,6 +211,12 @@ namespace Sequencer { if ( has_mode ) { this->is_acam_acquiring.store( acquiring ); } + // recorded per status, so a wait that starts after the change was already + // made still sees that it was raised and then cleared + if ( has_goalshift ) { + if ( goalshift ) this->acam_goalshift_raised_us.store( pubtime ); + else this->acam_goalshift_cleared_us.store( pubtime ); + } this->acam_pubtime.store( pubtime ); this->acam_cv.notify_all(); } @@ -944,12 +957,13 @@ namespace Sequencer { // re-applied during each readout (flexure_reapply, in the readout wait). // std::future flexure_future; + int64_t offset_sent_us = 0; // set when the target offset went to acamd as a goal change if ( !this->target.iscal ) { // send offsets only on fineacquire, otherwise user needs to fix things if ( !this->is_repeat_target.load() && this->is_fineacquire_locked.load() && - this->target_offset() == ERROR ) { + this->target_offset( &offset_sent_us ) == ERROR ) { if (this->wait_for_user()==ABORT) { this->broadcast.notice( function, "cancelled" ); return; @@ -968,9 +982,25 @@ namespace Sequencer { } } - // The offset-star offset is applied asynchronously by the ACAM guide loop; - // ACAMD_OFFSETGOAL returns before the move lands. Wait so the target is on - // the slit before exposing. Skipped when the target has no offset. + // A target offset sent while ACAM guides only changes acamd's goal. The + // guide loop makes the move with its next correction, whose pt_offset + // returns when the TCS answers the move. Expose once acamd reports that + // move made; the wait overlaps the slit move above and is bounded. If the + // move is not reported in time, expose anyway with a warning -- what the + // fixed settle always did -- so this can never hold the sequence. + if ( offset_sent_us > 0 ) { + const long offset_made = this->wait_for_target_offset( offset_sent_us ); + if ( offset_made == ABORT ) { + this->broadcast.notice( function, "cancelled" ); + return; + } + if ( offset_made != NO_ERROR ) { + this->broadcast.warning( function, "target offset not confirmed by acamd in time: exposing anyway" ); + } + } + + // Optional extra settle once the offset has been made (OFFSET_SETTLE_SEC, + // 0 = none). Skipped when the target has no offset. if ( !this->target.iscal && this->is_fineacquire_locked.load() && ( this->target.offset_ra != 0.0 || this->target.offset_dec != 0.0 ) && this->offset_settle_sec > 0.0 ) { @@ -4354,9 +4384,11 @@ namespace Sequencer { * @return ERROR or NO_ERROR * */ - long Sequence::target_offset() { + long Sequence::target_offset( int64_t *acam_sent_us ) { const std::string function("Sequencer::Sequence::target_offset"); + if ( acam_sent_us ) *acam_sent_us = 0; + // nothing to do if both ra and dec offsets are zero if (this->target.offset_ra == 0.0 && this->target.offset_dec == 0.0) return NO_ERROR; @@ -4371,7 +4403,11 @@ namespace Sequencer { const double ddec_deg = this->target.offset_dec / 3600.0; std::stringstream cmd; cmd << ACAMD_OFFSETGOAL << " " << std::fixed << std::setprecision(6) << dra_deg << " " << ddec_deg; + // taken before the send, so acamd's publish of the raised goal change is + // always later (no guard band needed, unlike do_acam_acquire) + const int64_t sent_us = get_time_us(); error = this->acamd.command( cmd.str() ); + if ( error == NO_ERROR && acam_sent_us ) *acam_sent_us = sent_us; } else // if ACAM is not guiding then I send the offsets directly to the TCS @@ -4390,6 +4426,61 @@ namespace Sequencer { /***** Sequencer::Sequence::target_offset ***********************************/ + /***** Sequencer::Sequence::wait_for_target_offset **************************/ + /** + * @brief wait until acamd has made a target offset sent as a goal change + * @details ACAMD_OFFSETGOAL only changes the guide goal. acamd's guide loop + * makes the move with its next correction, and that correction's + * pt_offset returns when the TCS answers the move (later for + * bigger moves). acamd raises GOALSHIFT_PENDING with the goal + * change and clears it once that correction has been sent and + * answered. The move is made once a status published after the + * send has shown it raised and a later one has shown it cleared. + * handletopic_acamd records both publish times as statuses + * arrive, so this holds even if the move was made before this + * wait began (the wait follows the slit move). Abort wakes acam_cv. + * @param[in] sent_us get_time_us() taken just before the offset was sent + * @return NO_ERROR | ERROR (not reported within TIMEOUT_SEC) | ABORT (cancelled) + * + */ + long Sequence::wait_for_target_offset( const int64_t sent_us ) { + const std::string function("Sequencer::Sequence::wait_for_target_offset"); + + // The move is made at acamd's next ACAM frame and answered by the TCS + // within seconds (measured: <= 4.3 s up to 147"); this covers a second + // frame and a 300" move. Bounded: the caller exposes either way. + // + constexpr double TIMEOUT_SEC = 30.0; + + logwrite( function, "NOTICE: waiting for acamd to make the target offset before exposing" ); + + const auto deadline = std::chrono::steady_clock::now() + std::chrono::duration( TIMEOUT_SEC ); + bool made = false; + { + std::unique_lock lock( this->acam_mtx ); + made = this->acam_cv.wait_until( lock, deadline, [&]() { + if ( this->cancel_flag.load() ) return true; + const int64_t raised = this->acam_goalshift_raised_us.load(); + return ( raised > sent_us && this->acam_goalshift_cleared_us.load() > raised ); + } ); + } + + if ( this->cancel_flag.load() ) return ABORT; + + std::ostringstream oss; + if ( !made ) { + oss << "ERROR acamd did not make the target offset within " << TIMEOUT_SEC << " s"; + logwrite( function, oss.str() ); + return ERROR; + } + oss << "target offset made by acamd " << std::fixed << std::setprecision(1) + << ( get_time_us() - sent_us ) / 1.0e6 << " s after it was sent"; + logwrite( function, oss.str() ); + return NO_ERROR; + } + /***** Sequencer::Sequence::wait_for_target_offset **************************/ + + /***** Sequencer::Sequence::publish_targetinfo *****************************/ /** * @brief publish target info on Topic::TARGETINFO, on change (or force) diff --git a/sequencerd/sequence.h b/sequencerd/sequence.h index 67978046..97328b4f 100644 --- a/sequencerd/sequence.h +++ b/sequencerd/sequence.h @@ -305,6 +305,8 @@ namespace Sequencer { std::atomic is_acam_guiding{false}; ///< is acam guiding (IS_ACQUIRED)? std::atomic is_acam_acquiring{false}; ///< is acam in an acquire mode? std::atomic acam_pubtime{0}; ///< publish time (us) of latest received acamd status + std::atomic acam_goalshift_raised_us{0}; ///< publish time (us) of the latest acamd status showing GOALSHIFT_PENDING set + std::atomic acam_goalshift_cleared_us{0}; ///< publish time (us) of the latest acamd status showing GOALSHIFT_PENDING clear /** @brief guard-band (us) subtracted from the acquire-command send time * when computing the freshness boundary in do_acam_acquire. Tolerates @@ -350,7 +352,7 @@ namespace Sequencer { tcs_settle_stable(1), tcs_domeazi_ready(1), tcs_preauth_time(0), - offset_settle_sec(3), + offset_settle_sec(0), do_once(false), tcs_which("real"), tcs_name("offline"), @@ -428,7 +430,7 @@ namespace Sequencer { double tcs_settle_stable; ///< time that TCS must report TRACKING before it is really tracking double tcs_domeazi_ready; ///< max degrees azimuth that dome and telescope can differ before ready to observe double tcs_preauth_time; ///< seconds before end of exposure to notify TCS of next target's coords (0 to disable) - double offset_settle_sec; ///< sec to wait after a target offset before exposing (config OFFSET_SETTLE_SEC; 0 disables) + double offset_settle_sec; ///< extra sec to wait after the target offset has been made, before exposing (config OFFSET_SETTLE_SEC; 0 = none) bool tcs_auto_ontarget = true; ///< TCS ?ONTARGET may satisfy the TCSOP wait (config TCS_AUTO_ONTARGET; default yes) double tcs_auto_ontarget_sep = 5.0; ///< max |telescope-target| arcsec for auto-ontarget (config TCS_AUTO_ONTARGET_SEP) double tcs_auto_ontarget_cass = 0.05; ///< max |TCS cass angle - commanded ring angle| deg for auto-ontarget (config TCS_AUTO_ONTARGET_CASS) @@ -621,7 +623,8 @@ namespace Sequencer { long get_tcs_weather_coords( double &ra_h, double &dec_d ); ///< read the current TCS ra,dec in decimal hr,deg long get_tcs_coords( double &ra_h, double &dec_d ); ///< read the current TCS ra,dec,cass in decimal hr,deg long get_tcs_cass( double &cass ); - long target_offset(); + long target_offset( int64_t *acam_sent_us = nullptr ); ///< acam_sent_us: send time if the offset went to acamd as a goal change, else 0 + long wait_for_target_offset( const int64_t sent_us ); ///< wait until acamd has made that goal change void publish_targetinfo( bool force=false ); ///< publish target info on change (or force) diff --git a/slicecamd/slicecam_interface.cpp b/slicecamd/slicecam_interface.cpp index ec21083e..0ee2da88 100644 --- a/slicecamd/slicecam_interface.cpp +++ b/slicecamd/slicecam_interface.cpp @@ -60,6 +60,10 @@ namespace Slicecam { const bool was_running = this->is_fineacquire_running.load(std::memory_order_acquire); this->is_fineacquire_locked.store(false, std::memory_order_release); this->is_fineacquire_running.store(false, std::memory_order_release); + { + std::lock_guard lock(this->acam_mtx); + this->acam_cv.notify_all(); + } this->publish_status(); logwrite(function, was_running ? "stop requested" : "stopped"); retstring=this->is_fineacquire_running.load(std::memory_order_acquire)?"running":"stopped"; @@ -419,6 +423,39 @@ namespace Slicecam { const int max_samples = this->fineacquire_state.max_samples; const int min_samples = this->fineacquire_state.min_samples; + // Verification pass, accept-only early exit: once a correction has been + // applied this run, a single frame already inside the goal is sufficient + // proof. A frame may only ACCEPT -- anything larger falls through to the + // usual median machinery, so a noisy frame can never trigger a + // correction or an extra cycle. + // + if ( this->fineacq_total_dra != 0.0 || this->fineacq_total_ddec != 0.0 ) { + const double r1 = std::hypot( offsets.first, offsets.second ) * 3600.0; + if ( r1 <= this->fineacquire_state.goal_arcsec ) { + std::ostringstream oss; + oss << "fine acquisition converged: single-frame offset r=" << r1 + << " arcsec <= goal=" << this->fineacquire_state.goal_arcsec + << " arcsec (n=" << n << ")"; + logwrite( function, oss.str() ); + + std::ostringstream acqmodel; + acqmodel << "[ACQMODEL] acam2slit dRA=" << this->fineacq_total_dra + << " dDEC=" << this->fineacq_total_ddec << " arcsec" + << " GOALRA=" << this->fineacq_goal_ra + << " GOALDEC=" << this->fineacq_goal_dec + << " CASANGLE=" << this->telem.angle_scope + << " n=" << n + << " cam=" << which; + logwrite( function, acqmodel.str() ); + + this->is_fineacquire_locked.store( true, std::memory_order_release ); + this->is_fineacquire_running.store( false, std::memory_order_release ); + this->fineacquire_state.reset(); + this->publish_status(); + return; + } + } + // wait for the minimum number of samples before evaluating anything // if ( n < min_samples ) return; @@ -562,6 +599,9 @@ namespace Slicecam { const double cmd_dra = effective_gain * med_dra; const double cmd_ddec = effective_gain * med_ddec; + const int64_t send_time = get_time_us(); + const bool was_guiding = this->is_acam_guiding.load(); + if ( this->offset_acam_goal( { cmd_dra, cmd_ddec }, true ) != NO_ERROR ) { logwrite( function, "ERROR failed to send offset to ACAM" ); this->is_fineacquire_running.store( false, std::memory_order_release ); @@ -569,6 +609,36 @@ namespace Slicecam { return; } + // while guiding the correction only re-points acamd's goal; wait until + // acamd reports the goal change applied before settling + if ( was_guiding && this->fineacquire_state.move_timeout_sec > 0.0 ) { + std::unique_lock lock(this->acam_mtx); + // completion requires seeing THIS request registered (pending true on a + // status after the send) and then cleared; statuses arrive in publish + // order, so the false cannot predate the request + bool seen_pending = false; + const bool applied = this->acam_cv.wait_for( lock, + std::chrono::duration( this->fineacquire_state.move_timeout_sec ), + [this, send_time, &seen_pending]() { + if ( !this->is_fineacquire_running.load(std::memory_order_acquire) || + !this->should_framegrab_run.load(std::memory_order_acquire) ) return true; + if ( this->last_acam_pubtime.load(std::memory_order_acquire) <= send_time ) return false; + if ( this->acam_goalshift_pending.load(std::memory_order_acquire) ) { + seen_pending = true; + return false; + } + return seen_pending; + }); + if ( !this->should_framegrab_run.load(std::memory_order_acquire) ) return; + if ( !this->is_fineacquire_running.load(std::memory_order_acquire) ) return; + if ( !applied ) { + std::ostringstream w; + w << "WARNING goal change not applied within " + << this->fineacquire_state.move_timeout_sec << " s; proceeding"; + logwrite( function, w.str() ); + } + } + // time-based settle: wait for the TCS to physically finish the move before // sampling resumes. The frame-count settle (settle_frames) is too short in // wall-clock when autoexpose shortens the exposure (e.g. bright targets), so @@ -924,6 +994,10 @@ namespace Slicecam { Common::extract_telemetry_value( jmessage, Key::PUBTIME, pubtime ); this->last_acam_pubtime.store( pubtime, std::memory_order_relaxed ); + bool goalshift_pending=false; + Common::extract_telemetry_value( jmessage, Key::Acamd::GOALSHIFT_PENDING, goalshift_pending ); + this->acam_goalshift_pending.store( goalshift_pending, std::memory_order_relaxed ); + // wake any thread waiting on ACAM state (e.g. fineacquire) std::lock_guard lock(this->acam_mtx); this->acam_cv.notify_all(); @@ -1418,6 +1492,19 @@ namespace Slicecam { applied++; } else + if ( config.param[entry] == "FINE_ACQUIRE_MOVE_TIMEOUT" ) { + try { this->fineacquire_state.move_timeout_sec = std::stod( config.arg[entry] ); } + catch ( const std::exception &e ) { + message.str(""); message << "ERROR invalid FINE_ACQUIRE_MOVE_TIMEOUT " + << config.arg[entry] << ": " << e.what(); + logwrite( function, message.str() ); + return ERROR; + } + message.str(""); message << "SLICECAMD:config:" << config.param[entry] << "=" << config.arg[entry]; + logwrite( function, message.str() ); + applied++; + } + else if ( config.param[entry] == "FINE_ACQUIRE_GAIN" ) { try { this->fineacquire_state.gain = std::stod( config.arg[entry] ); } catch ( const std::exception &e ) { @@ -2101,6 +2188,10 @@ namespace Slicecam { // if ( whattodo == "stop" ) { this->should_framegrab_run.store( false, std::memory_order_release ); // tells framegrab loop to stop + { + std::lock_guard lock(this->acam_mtx); + this->acam_cv.notify_all(); + } if ( this->is_framegrab_running.load(std::memory_order_acquire) ) { // wait for it to stop int wait_ms = std::max( static_cast(3000*(this->camera.andor.begin()->second->camera_info.exptime+1)), 5000 ); // alert user that framegrabbing has stopped @@ -2180,6 +2271,10 @@ namespace Slicecam { // frame will be grabbed. // this->should_framegrab_run.store( false, std::memory_order_release ); + { + std::lock_guard lock(this->acam_mtx); + this->acam_cv.notify_all(); + } if ( this->is_framegrab_running.load(std::memory_order_acquire) ) return; } else @@ -2596,6 +2691,10 @@ namespace Slicecam { // bool is_guiding = this->is_acam_guiding.load(); + // arcsec for the log; the guiding path sends degrees + const double ra_off_arcsec = ra_off * 3600.; + const double dec_off_arcsec = dec_off * 3600.; + // send the offsets now // if ( is_guiding ) { @@ -2632,7 +2731,7 @@ namespace Slicecam { } std::ostringstream message; - message << "requested offsets dRA=" << ra_off << " dDEC=" << dec_off << " arcsec"; + message << "requested offsets dRA=" << ra_off_arcsec << " dDEC=" << dec_off_arcsec << " arcsec"; logwrite(function, message.str()); return NO_ERROR; diff --git a/slicecamd/slicecam_interface.h b/slicecamd/slicecam_interface.h index 90a7fdb1..b3e018c6 100644 --- a/slicecamd/slicecam_interface.h +++ b/slicecamd/slicecam_interface.h @@ -95,6 +95,7 @@ namespace Slicecam { int settle_frames = 0; ///< countdown of frames to discard while telescope settles int settle_count = 2; ///< configured: frames to discard after each move double settle_sec = 3.0; ///< time-based settle (sec) after each move; 0 disables (needed when autoexpose shortens frames so the frame-count settle is too brief in wall-clock) + double move_timeout_sec = 60.0; ///< max wait (sec) for acamd to report the goal change applied; 0 disables the wait int consecutive_centroid_failures = 0; ///< counts consecutive centroid failures // exposure compensation (shared by the reactive trim and, later, autoexpose) double exptime_min = 0.1; ///< clamp: minimum auto-adjusted exposure (sec) @@ -193,6 +194,7 @@ namespace Slicecam { std::atomic is_acam_guiding; ///< is acam guiding? std::atomic last_acam_pubtime{0}; ///< pubtime (us) of latest received acamd status + std::atomic acam_goalshift_pending{false}; ///< acamd has an unexecuted goal change // Latest target (goal) coords published on Topic::TARGETINFO // NAN until a TARGETINFO arrives, so manual runs with no sequencer target log nan.