diff --git a/Config/acamd.cfg.in b/Config/acamd.cfg.in index 3b101dee..8153adf1 100644 --- a/Config/acamd.cfg.in +++ b/Config/acamd.cfg.in @@ -157,10 +157,10 @@ MOTOR_POS="cover 1 1 504 close" # ACQUIRE_TIMEOUT=90 # seconds before ACAM acquisition sequence aborts on failure to acquire (REQUIRED!) ACQUIRE_RETRYS=5 # max number of retrys before acquisition fails (optional, can be left blank to disable) -ACQUIRE_OFFSET_THRESHOLD=0.5 # computed offset below this threshold (in arcsec) defines successful acquisition +ACQUIRE_OFFSET_THRESHOLD=1.0 # computed offset below this threshold (in arcsec) defines successful acquisition ACQUIRE_MIN_REPEAT=2 # minimum number of sequential successful acquires ACQUIRE_TCS_MAX_OFFSET=60 # max offset (arcsec) for an ordinary guiding correction sent to the TCS -ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET=300 # max offset (arcsec) for a deliberate goal offset (put-on-slit, offset-star, end-of-fineacquire, pyGUI 'Offset') applied while guiding +ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET=300 # max offset (arcsec) for the one correction that makes a deliberate goal offset (put-on-slit, offset-star, end-of-fineacquire, pyGUI 'Offset'); guide and fine-acquire corrections keep ACQUIRE_TCS_MAX_OFFSET # SkySimulator options: # SKYSIM_IMAGE_SIZE= where is integer diff --git a/Config/sequencerd.cfg.in b/Config/sequencerd.cfg.in index e665eeb6..b50de05b 100644 --- a/Config/sequencerd.cfg.in +++ b/Config/sequencerd.cfg.in @@ -150,6 +150,9 @@ TCS_OFFSET_RATE_RA=45 # TCS offset rate for RA ("MRATE") in arcse TCS_OFFSET_RATE_DEC=45 # TCS offset rate for DEC ("MRATE") in arcsec per second TCS_SETTLE_TIMEOUT=20 # seconds to wait for telescope to settle (for "TRACKING_STABLY") TCS_SETTLE_STABLE=1.5 # time in fractional seconds (to 0.1) that TCS must report TRACKING before it is really stable +TCS_AUTO_ONTARGET=yes # yes: TCS ?ONTARGET=1 + tracking + sep < TCS_AUTO_ONTARGET_SEP + cass ring within TCS_AUTO_ONTARGET_CASS (2 consecutive 1 Hz reads) satisfies the ontarget wait; operator press always works +TCS_AUTO_ONTARGET_SEP=5.0 # arcsec: max |telescope-target| for auto-ontarget +TCS_AUTO_ONTARGET_CASS=0.05 # deg: max |TCS cass angle - commanded ring angle| for auto-ontarget TCS_DOMEAZI_READY=10 # max degrees azimuth that dome and telescope can differ before ready to observe TCS_PREAUTH_TIME=10 # seconds before end of exposure to notify TCS of next target's coords (0 to disable) @@ -159,7 +162,7 @@ ACQUIRE_TIMEOUT=90 # seconds before ACAM acquisition sequence ACQUIRE_RETRYS=5 # max number of retrys before acquisition fails (optional, can be left blank to disable) ACQUIRE_OFFSET_THRESHOLD=0.5 # computed offset below this threshold (in arcsec) defines successful acquisition ACQUIRE_MIN_REPEAT=2 # minimum number of sequential successful acquires -OFFSET_SETTLE_SEC=3 # sec to wait after an offset-star target offset before exposing (0 disables; only delays targets with a non-zero offset) +OFFSET_SETTLE_SEC=0 # extra sec to wait before exposing once acamd has made an offset-star target offset (the sequencer waits for the move itself; 0 = no extra wait) ACQUIRE_TCS_MAX_OFFSET=60 # the maximum allowable offset sent to the TCS, in arcsec # Calibration Settings diff --git a/Config/slicecamd.cfg.in b/Config/slicecamd.cfg.in index 3bd6316a..6bd4d003 100644 --- a/Config/slicecamd.cfg.in +++ b/Config/slicecamd.cfg.in @@ -86,17 +86,46 @@ FINE_ACQUIRE_MIN_SAMPLES=3 # FINE_ACQUIRE_SETTLE_FRAMES= # frames discarded after each telescope move to allow settling before -# centroid measurements resume. +# centroid measurements resume. The wall-clock settle (FINE_ACQUIRE_SETTLE_SEC, +# applied after acamd confirms the goal change) covers the move itself. # -FINE_ACQUIRE_SETTLE_FRAMES=2 +FINE_ACQUIRE_SETTLE_FRAMES=0 # FINE_ACQUIRE_SETTLE_SEC= -# Time-based settle applied after each move, in addition to SETTLE_FRAMES, -# to let the TCS physically finish the move before sampling resumes. This matters -# when autoexpose shortens the exposure (e.g. bright targets): the frame-count -# settle then elapses too quickly in wall-clock. Set to 0 to disable. -# -FINE_ACQUIRE_SETTLE_SEC=3.0 +# Pause after each correction, before the next exposure starts. The frame +# loop already waits for acamd to report the move made, which is only after the +# TCS has answered it (#479), so no frame can overlap the move. acamd usually +# reports 1 s after the answer, but sooner if anything requests its status in +# that second; this pause guarantees the settle either way. Set to 0 to disable. +# +FINE_ACQUIRE_SETTLE_SEC=1 + +# FINE_ACQUIRE_MAX_ADJUSTMENTS= +# Correction budget. At most corrections are commanded in one run, and the +# -th decides. It is sent only if the offset it removes is within +# FINE_ACQUIRE_RELAXED_FACTOR x goal; the run then converges once that move has +# executed and settled ("converged (FINAL CORRECTION)"). If the offset is +# larger, the run fails at once without sending it ("fine acquisition FAILED: +# correction of would be the last ..."). A correction acamd does not +# confirm within FINE_ACQUIRE_MOVE_TIMEOUT also fails a budgeted run. The +# sequencer reports a failure as a failed acquisition. Nothing is sampled or +# evaluated once no correction can follow. Needs FINE_ACQUIRE_MOVE_TIMEOUT > 0. +# 0 disables (correct until the goal or the sequencer timeout). +FINE_ACQUIRE_MAX_ADJUSTMENTS=3 + +# FINE_ACQUIRE_RELAXED_FACTOR= +# Largest offset, as a multiple of the 0.4" fine-acquire goal, that the budget's +# last correction may remove: factor 2.0 sends it, and then converges, only if +# that offset is <= 0.8". Must be >= 1 (1 means the last correction is never sent). +FINE_ACQUIRE_RELAXED_FACTOR=2.0 + +# FINE_ACQUIRE_MOVE_TIMEOUT= +# Max time to wait for acamd to report a requested goal change applied +# before the settle begins. On timeout the settle proceeds anyway, except under +# a correction budget, where the run fails. Set to 0 to disable the wait (not +# allowed with FINE_ACQUIRE_MAX_ADJUSTMENTS > 0). +# +FINE_ACQUIRE_MOVE_TIMEOUT=60.0 # FINE_ACQUIRE_GAIN= # Proportional gain = {0..1} applied to the commanded offset when the residual diff --git a/acamd/acam_interface.cpp b/acamd/acam_interface.cpp index f3a0daf1..7e0b47c9 100644 --- a/acamd/acam_interface.cpp +++ b/acamd/acam_interface.cpp @@ -1443,6 +1443,7 @@ namespace Acam { const int attempts = this->target.attempts; const std::string filter = this->motion.get_current_filtername(); const std::string cover = this->motion.get_current_coverpos(); + const bool goalshift_pending = this->target.goalshift_pending.load(); // unless forced, only publish if there was a change in any one of these // @@ -1452,7 +1453,8 @@ namespace Acam { nacquired == this->last_status.nacquired && attempts == this->last_status.attempts && filter == this->last_status.filter && - cover == this->last_status.cover ) return; + cover == this->last_status.cover && + goalshift_pending == this->last_status.goalshift_pending ) return; this->last_status.acquire_mode = acquire_mode; this->last_status.is_acquired = is_acquired; @@ -1460,6 +1462,7 @@ namespace Acam { this->last_status.attempts = attempts; this->last_status.filter = filter; this->last_status.cover = cover; + this->last_status.goalshift_pending = goalshift_pending; // assemble the telemetry into a json message // @@ -1473,6 +1476,7 @@ namespace Acam { jmessage_out[Key::Acamd::BACKGROUND] = this->astrometry.get_background(); jmessage_out[Key::Acamd::FILTER] = filter; jmessage_out[Key::Acamd::COVER] = cover; + jmessage_out[Key::Acamd::GOALSHIFT_PENDING] = goalshift_pending; jmessage_out[Key::PUBTIME] = get_time_us(); try { @@ -3221,13 +3225,15 @@ logwrite( function, "[DEBUG] acquire here" ); } this->astrometry.solve( last_imagename, this->target.ext_solver_args ); - this->astrometry.get_solution( result, this->target.acam_goal.ra, this->target.acam_goal.dec, this->target.acam_goal.angle, matches, rmsarcsec ); + double solved_ra=NAN, solved_dec=NAN, solved_angle=NAN; + this->astrometry.get_solution( result, solved_ra, solved_dec, solved_angle, matches, rmsarcsec ); + this->target.set_goal( solved_ra, solved_dec, solved_angle ); // under goal_mtx: the guide loop may be reading it if ( result=="NOISY" || result=="GOOD" ) { goodsolution=true; - message.str(""); message << "setting goals from solver: " << this->target.acam_goal.ra << " " - << this->target.acam_goal.dec << " " - << this->target.acam_goal.angle; + message.str(""); message << "setting goals from solver: " << solved_ra << " " + << solved_dec << " " + << solved_angle; logwrite( function, message.str() ); break; } @@ -3307,11 +3313,13 @@ logwrite( function, message.str() ); message.str(""); message << "[DEBUG] input coords_slit.ra="<< this->target.coords_slit.ra << " .dec=" << this->target.coords_slit.dec; logwrite( function, message.str() ); + double goal_ra=NAN, goal_dec=NAN, goal_angle=NAN; this->fpoffsets.compute_offset_last_angle( this->target.get_pointmode(), "ACAM", this->target.coords_slit.ra, this->target.coords_slit.dec, - this->target.acam_goal.ra, this->target.acam_goal.dec, this->target.acam_goal.angle ); + goal_ra, goal_dec, goal_angle ); + this->target.set_goal( goal_ra, goal_dec, goal_angle ); // under goal_mtx: the guide loop may be reading it -message.str(""); message << "[DEBUG] goals: " << this->target.acam_goal.ra << " " << this->target.acam_goal.dec << " " << this->target.acam_goal.angle; +message.str(""); message << "[DEBUG] goals: " << goal_ra << " " << goal_dec << " " << goal_angle; logwrite( function, message.str() ); if ( error == NO_ERROR ) { @@ -3392,7 +3400,10 @@ logwrite( function, message.str() ); this->nacquired = 0; this->attempts = 0; this->sequential_failures = 0; - this->allow_large_offset.store(false); // no stale deliberate-offset allowance + { + std::lock_guard lock(this->goal_mtx); + this->deliberate_gen = 0; // no stale deliberate-offset allowance + } this->is_acquired.store( false, std::memory_order_release ); // Start the timeout clock, initialized as the time now plus the @@ -3466,6 +3477,7 @@ logwrite( function, message.str() ); // an attempt and will keep trying until max attempts. // long error = NO_ERROR; + bool frame_superseded = false; // the goal changed while this frame's correction was computed if ( this->acquire_mode == Acam::TARGET_ACQUIRE || this->acquire_mode == Acam::TARGET_ACQUIRE_HERE || @@ -3521,8 +3533,11 @@ message.str(""); message << "[DEBUG] ACQUIRE_HERE using goals: " << this->acam_g logwrite( function, message.str() ); } ******/ -message.str(""); message << "[DEBUG] using goals: " << this->acam_goal.ra << " " << this->acam_goal.dec << " " << this->acam_goal.angle; +{ +const goal_snapshot_t logged = this->snapshot_goal(); +message.str(""); message << "[DEBUG] using goals: " << logged.ra << " " << logged.dec << " " << logged.angle; logwrite( function, message.str() ); +} message.str(""); message << "[DEBUG] dRA=" << iface->target.dRA << " dDEC=" << iface->target.dDEC; logwrite( function, message.str() ); @@ -3586,7 +3601,8 @@ logwrite( function, message.str() ); // // Offsets calculated as difference between acam goals and the solution from // solve. ACAM goals are the SLIT->ACAM translated coords from the DB and - // are where the ACAM needs to be pointed. Goals never change. + // are where the ACAM needs to be pointed. Goals change only through + // offset_goal and a new acquisition (set_goal); see goal_mtx. // double ra_off, dec_off; // calculated offsets will be in degrees @@ -3609,13 +3625,31 @@ logwrite( function, message.str() ); offset = angular_separation( acam_goal.ra, acam_goal.dec, acam_ra, acam_dec ); } *****/ + // Every correction is classified by the goal it is computed against, read + // together with that goal's generation (PAL-171; see goal_mtx): + // + // - guide correction: the loop has already moved to this goal. Capped at + // ACQUIRE_TCS_MAX_OFFSET and, while guiding, median-filtered. It makes + // no goal change and spends nothing that belongs to one. + // - goal-change correction: offset_goal changed the goal and no correction + // computed against it has been sent. A fine-acquire change + // ('fineguiding') keeps the ACQUIRE_TCS_MAX_OFFSET cap; a deliberate + // change among those still to be made allows + // ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET. Sent at once and alone, never + // through the median filter. Once sent it has made the change(s) and + // clears goalshift_pending, unless a newer change arrived meanwhile. + // + // So a guide correction in flight when a deliberate offset arrives can + // neither use nor spend the deliberate cap. + // + const goal_snapshot_t goal = this->snapshot_goal(); if ( iface->fpoffsets.solve_offset( acam_ra, acam_dec, - this->acam_goal.ra, this->acam_goal.dec, + goal.ra, goal.dec, ra_off, dec_off ) == ERROR ) break; - offset = angular_separation( this->acam_goal.ra, this->acam_goal.dec, acam_ra, acam_dec ); + offset = angular_separation( goal.ra, goal.dec, acam_ra, acam_dec ); message.str(""); message << "[DEBUG] acam_ra=" << acam_ra << " acam_dec=" << acam_dec << " acam_goal.ra=" - << acam_goal.ra << " .dec=" << this->acam_goal.dec << " .ang=" << this->acam_goal.angle; + << goal.ra << " .dec=" << goal.dec << " .ang=" << goal.angle; logwrite( function, message.str() ); message.str(""); message << "[DEBUG] calculated RAoffset=" << ra_off << " DECLoffset=" << dec_off; logwrite( function, message.str() ); @@ -3634,18 +3668,18 @@ logwrite( function, message.str() ); // There is a maximum offset we send to the TCS. This is not a TCS limit // (theirs is very large); it is our safety limit so that a bad solution - // can't move us far off the slit. Ordinary guiding corrections use the - // normal tcs_max_offset (ACQUIRE_TCS_MAX_OFFSET). A deliberate goal offset - // applied while guiding -- via offset_goal, which covers put-on-slit, - // offset-star acquisition, the end-of-fineacquire target offset, and the - // pyGUI 'Offset' button -- is intentionally larger, so for the one - // correction that consumes it we allow up to tcs_max_putonslit_offset - // (ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET). The ACQUIRE path uses tcs_max_offset - // and is unaffected. + // can't move us far off the slit. Guide and fine-acquire corrections use + // tcs_max_offset (ACQUIRE_TCS_MAX_OFFSET). Only the correction that makes + // a deliberate goal offset -- put-on-slit, offset-star acquisition, the + // end-of-fineacquire target offset, the pyGUI 'Offset' button -- may move + // up to tcs_max_putonslit_offset (ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET). // - double maxoffset = this->tcs_max_offset; - if ( this->acquire_mode == Acam::TARGET_GUIDE && this->allow_large_offset.load() ) { - maxoffset = this->tcs_max_putonslit_offset; + const double maxoffset = goal.deliberate ? this->tcs_max_putonslit_offset : this->tcs_max_offset; + + if ( goal.goal_change ) { + message.str(""); message << "NOTICE correction for goal change " << goal.gen << ": offset=" << offset + << " arcsec, cap " << maxoffset << " arcsec"; + logwrite( function, message.str() ); } // Check the requested offset against the applicable max allowed offset @@ -3654,13 +3688,14 @@ logwrite( function, message.str() ); message.str(""); message << "[WARNING] calculated offset " << offset << " not below max " << maxoffset << " and will not be sent to the TCS"; logwrite( function, message.str() ); - this->allow_large_offset.store(false); // deliberate-offset allowance consumed even when rejected + // A refused correction has made no goal change: a deliberate change + // keeps its cap for the next frame. // Match found but failure to send an offset is considered an attempt // so attempts is incremented. // if ( attempts >= this->max_attempts ) { - message.str(""); message << "ERROR: failed to find offset below " << this->tcs_max_offset << " within max number of attempts"; + message.str(""); message << "ERROR: failed to find offset below " << maxoffset << " within max number of attempts"; iface->async.enqueue_and_log( function, message.str() ); error = ERROR; break; @@ -3678,7 +3713,29 @@ logwrite( function, message.str() ); bool should_offset=false; - // While guiding, run the offsets through a median filter over a + // A correction computed against a goal that has changed since is + // neither sent nor filtered: the next frame is computed against the new + // goal. + // + if ( !this->goal_unchanged_since( goal.gen ) ) { + // Measured against a goal that no longer exists: not sent, and the + // frame counts as neither an acquisition nor a failure. + logwrite( function, "NOTICE goal changed while this correction was computed: not sent" ); + attempts = 0; + frame_superseded = true; + break; + } + else + // A goal-change correction is sent at once and alone. It never enters + // the median filter, and the samples there, computed against the old + // goal, are discarded. + // + if ( goal.goal_change ) { + this->reset_offset_params(); + should_offset = true; + } + else + // While guiding, run guide corrections through a median filter over a // specified time period, and send the offsets only at the rate // determined by that period. This returns true and modifies the // values ra_off,dec_off when it's time to offset. @@ -3691,7 +3748,10 @@ logwrite( function, message.str() ); if ( should_offset ) { // send offset to TCS here (returns when offset is complete) if ( iface->tcsd.pt_offset( ra_off*3600., dec_off*3600., OFFSETRATE )==ERROR) break; - this->allow_large_offset.store(false); // deliberate-offset allowance consumed + // A goal-change correction has now made its change(s). A guide + // correction makes none: a goal change that arrived while it was in + // flight keeps its own cap and stays pending. + if ( goal.goal_change ) this->goal_change_made( goal.gen ); std::this_thread::sleep_for( std::chrono::seconds(1) ); } @@ -3727,7 +3787,7 @@ logwrite( function, message.str() ); } while ( false ); // the do-loop is executed only once. used to allow breaks. } // end of acquisition sequence - if ( this->nacquired == 0 && this->max_attempts > 0 ) { + if ( !frame_superseded && this->nacquired == 0 && this->max_attempts > 0 ) { logwrite(function,"[DEBUG] nacquired="+std::to_string(nacquired)); if ( ++this->sequential_failures >= this->max_attempts ) { logwrite( function, "ERROR sequential failures "+std::to_string(this->sequential_failures) @@ -5474,8 +5534,9 @@ logwrite( function, message.str() ); /***** Acam::Interface::offset_goal *****************************************/ /** * @brief sets the dRA,dDEC offsets to apply to goal - * @details The offset is not applied here, this only stores the offsets - * in the class. Offsets are applied only while guiding. + * @details This changes the goal only. The acquisition or guide loop + * moves the telescope with its next correction computed against + * the new goal (see goal_mtx in Target). * @param[in] args input string expected " " * @param[out] retstring return string * @return ERROR | NO_ERROR @@ -5497,7 +5558,7 @@ logwrite( function, message.str() ); retstring = ACAMD_OFFSETGOAL; retstring.append( " [ [ fineguiding ] ]\n" ); retstring.append( " Apply offsets to the ACAM goal coordinates.\n" ); - retstring.append( " These offsets are applied only while guiding. If omitted,\n" ); + retstring.append( " The acquisition or guide loop then moves to the new goal. If omitted,\n" ); retstring.append( " the current offsets are returned. Units are in degrees.\n" ); retstring.append( " 'fineguiding' marks a slicecam fine-guiding correction (keeps the 60\" cap).\n" ); return HELP; @@ -5512,27 +5573,48 @@ logwrite( function, message.str() ); retstring="invalid_argument"; return ERROR; } - this->target.dRA = dRA; - this->target.dDEC = dDEC; - - // optional fineguiding flag: marks a slicecam fine-guiding correction, which + // optional fineguiding flag: marks a slicecam fine-acquire correction, which // must stay within the ordinary 60" guiding cap. Its absence means a // deliberate goal offset (put-on-slit, offset-star, end-of-fineacquire, // pyGUI 'Offset') that may legitimately be larger. std::string flag; const bool is_fineguiding = (iss >> flag && flag == "fineguiding"); - // Apply any dRA, dDEC goal offsets from the "put on slit" action to - // acam_ra_goal, acam_dec_goal. These dRA,dDEC offsets can come from - // either the ACAM or slicecam GUIs and are stored in the Target class. + // Apply the dRA,dDEC offsets to the goal and record the change with its + // kind, in one step under goal_mtx: the guide loop never reads half a + // change, and two offsets arriving on different connections are both kept. + // Only a correction the guide loop computes against this goal (or a later + // one) can make the change; for a deliberate offset that correction alone + // may move up to ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET, whatever mode the loop is + // in. A correction already in flight can neither use nor spend it (PAL-171). // - // The supplied acam_ra_goal,acam_dec_goal are modified by dRA,dDEC - // - this->fpoffsets.apply_offset( this->target.acam_goal.ra, this->target.dRA, - this->target.acam_goal.dec, this->target.dDEC ); + unsigned long long gen=0; + { + std::lock_guard lock( this->target.goal_mtx ); + double goal_ra = this->target.acam_goal.ra; + double goal_dec = this->target.acam_goal.dec; + if ( this->fpoffsets.apply_offset( goal_ra, dRA, goal_dec, dDEC ) == ERROR ) { + logwrite( function, "ERROR applying the offset to the goal: goal unchanged" ); + retstring="goal_unchanged"; + return ERROR; + } + this->target.acam_goal.ra = goal_ra; + this->target.acam_goal.dec = goal_dec; + this->target.dRA = dRA; + this->target.dDEC = dDEC; + gen = ++this->target.goal_gen; + if ( !is_fineguiding && ( dRA != 0.0 || dDEC != 0.0 ) ) { + this->target.deliberate_gen = gen; // a zero offset changes nothing, so it raises no cap + } + this->target.goalshift_pending.store( true, std::memory_order_release ); + } + this->publish_status(true); + message.str(""); message << "goal change " << gen << " (" << ( is_fineguiding ? "fine-acquire" : "deliberate offset" ) + << "): dRA=" << dRA*3600. << " dDEC=" << dDEC*3600. << " arcsec"; + logwrite( function, message.str() ); - message.str(""); message << this->target.dRA << " " << this->target.dDEC; + message.str(""); message << dRA << " " << dDEC; retstring = message.str(); // Applying an offset to the goal while guiding must never drop out of @@ -5540,14 +5622,8 @@ logwrite( function, message.str() ); // TARGET_GUIDE and reset the offset filter so the new goal takes effect // quickly. This covers GUI "put on slit" and sequencer target offsets. // - // This is a deliberate offset (put-on-slit, offset-star, end-of-fineacquire, - // or the pyGUI 'Offset' button) and may exceed the normal guiding cap, so - // allow the next correction up to ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET. The - // allowance is one-shot: do_acquire consumes it when the offset is sent. - // if ( this->target.acquire_mode == Acam::TARGET_GUIDE ) { this->target.reset_offset_params(); - if ( !is_fineguiding ) this->target.allow_large_offset.store(true); } this->publish_status(); diff --git a/acamd/acam_interface.h b/acamd/acam_interface.h index 4c2f59fd..4648bffa 100644 --- a/acamd/acam_interface.h +++ b/acamd/acam_interface.h @@ -349,9 +349,10 @@ namespace Acam { std::atomic is_acquired; ///< set if target acquired successfully std::atomic stop_acquisition; ///< set if the acquisition sequence should stop + std::atomic goalshift_pending{false}; ///< a requested goal change awaits execution by the guide loop double tcs_max_offset; - double tcs_max_putonslit_offset{300.}; ///< max offset (arcsec) for a deliberate goal offset (put-on-slit etc.) applied while guiding; defaults 300 if ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET absent + double tcs_max_putonslit_offset{300.}; ///< max offset (arcsec) for the correction that makes a deliberate goal offset (put-on-slit etc.); defaults 300 if ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET absent double offset_cal_offset, offset_cal_raoff, offset_cal_decoff; @@ -386,7 +387,14 @@ namespace Acam { this->time_offs.clear(); this->tcs_offset_period = val; } - inline void reset_offset_params() { this->reset_offset_params(this->tcs_offset_period); } + /// clear the filter only: never rewrite tcs_offset_period, which a concurrent + /// offsetperiod command may be setting + inline void reset_offset_params() { + std::lock_guard lock(this->offset_params_mtx); + this->ra_offs.clear(); + this->dec_offs.clear(); + this->time_offs.clear(); + } inline double get_timeout() { return this->timeout; } @@ -483,9 +491,72 @@ namespace Acam { double ra; double dec; double angle; - } acam_goal; + } acam_goal; ///< guarded by goal_mtx (snapshot_goal, set_goal, offset_goal) - std::atomic allow_large_offset{false}; ///< one-shot: allow the next guiding correction up to tcs_max_putonslit_offset + // Goal changes and the corrections that make them (PAL-171). + // + // offset_goal changes the goal; the guide loop moves the telescope. Every + // correction is classified by the goal it was computed against: + // guide correction -- a goal the loop has already moved to; + // goal-change correction -- a goal changed since: a fine-acquire + // correction ('fineguiding') or a deliberate + // offset (untagged: target offset, offset star, + // put-on-slit, pyGUI 'Offset'). + // goal_gen counts goal changes (offset_goal) and replacements (set_goal, + // which counts as made). goal_gen_applied is the newest change that a SENT + // correction was computed against. deliberate_gen is the newest + // deliberate change (0 = none); it is still to be made while + // deliberate_gen > goal_gen_applied, and only a correction computed against + // it (or a later goal) can make it -- so a correction already in flight can + // neither use nor spend its cap. All guarded by goal_mtx. + // + std::mutex goal_mtx; + unsigned long long goal_gen{0}; + unsigned long long goal_gen_applied{0}; + unsigned long long deliberate_gen{0}; + + struct goal_snapshot_t { + double ra, dec, angle; + unsigned long long gen; ///< goal generation the correction is computed against + bool goal_change; ///< includes a goal change not yet made + bool deliberate; ///< includes a deliberate goal change not yet made + }; + + /// the goal, its generation and its classification, read together + inline goal_snapshot_t snapshot_goal() { + std::lock_guard lock(this->goal_mtx); + return { this->acam_goal.ra, this->acam_goal.dec, this->acam_goal.angle, this->goal_gen, + this->goal_gen > this->goal_gen_applied, + this->deliberate_gen > this->goal_gen_applied }; + } + + /// true if no goal change has arrived since the snapshot of generation gen + inline bool goal_unchanged_since( unsigned long long gen ) { + std::lock_guard lock(this->goal_mtx); + return this->goal_gen == gen; + } + + /// a correction computed against generation gen has been sent: the goal + /// changes up to gen are made. goalshift_pending stays set if a newer one arrived. + inline void goal_change_made( unsigned long long gen ) { + std::lock_guard lock(this->goal_mtx); + if ( gen > this->goal_gen_applied ) this->goal_gen_applied = gen; + this->goalshift_pending.store( this->goal_gen > this->goal_gen_applied, std::memory_order_release ); + } + + /// a new acquisition replaces the goal (from the target coordinates or a + /// solve): every earlier change is void and nothing is pending. The + /// generation moves on, so a correction computed against the replaced goal + /// is dropped; the new goal counts as made (the acquisition moves to it). + inline void set_goal( const double ra, const double dec, const double angle ) { + std::lock_guard lock(this->goal_mtx); + this->acam_goal.ra = ra; + this->acam_goal.dec = dec; + this->acam_goal.angle = angle; + this->goal_gen_applied = ++this->goal_gen; + this->deliberate_gen = 0; + this->goalshift_pending.store( false, std::memory_order_release ); + } Target() : iface(nullptr), timeout(10), max_attempts(-1), min_repeat(1), is_acquired(false), @@ -530,6 +601,7 @@ namespace Acam { int attempts = 0; std::string filter = ""; std::string cover = ""; + bool goalshift_pending = false; } last_status; public: diff --git a/camerad/astrocam.cpp b/camerad/astrocam.cpp index 6a5792d1..13a12dd8 100644 --- a/camerad/astrocam.cpp +++ b/camerad/astrocam.cpp @@ -957,6 +957,10 @@ namespace AstroCam { long Interface::do_abort() { std::string function = "AstroCam::Interface::do_abort"; std::stringstream message; + // No sub-exposure follows an abort: clearing exposure_pending below wakes + // dothread_monitor_exposure_pending, which starts the next sub-exposure + // whenever nexp is still > 0 after its decrement. + this->nexp.store(0); // int this_expbuf = this->get_expbuf(); for ( const auto &dev : this->active_devnums ) { this->exposure_pending( dev, false ); @@ -1019,9 +1023,10 @@ namespace AstroCam { return ERROR; } - // Don't make any changes while frames are still being written. + // Don't make any changes while frames are still being written: give a + // write that is finishing (ms after READY) up to 2 s, then refuse. // - if ( this->any_writes_pending() ) { + if ( !this->wait_writes_done( 2000 ) ) { logwrite( function, "ERROR: cannot change binning while frame write is in progress" ); retstring="write_in_progress"; return ERROR; @@ -2713,7 +2718,7 @@ namespace AstroCam { if ( interface.nexp.load() > 0 ) { message.str(""); message << "starting next exposure. " << interface.nexp << " remaining"; interface.broadcast.notice( function, message.str() ); - interface.do_expose(interface.nexp); + interface.do_expose(interface.nexp, true); // continuation: nexp already holds what is left } else { interface.status.can_expose.store(true); @@ -2768,7 +2773,7 @@ namespace AstroCam { * @return ERROR or NO_ERROR * */ - long Interface::do_expose(int nexp_in) { + long Interface::do_expose(int nexp_in, bool continuation) { const std::string function("AstroCam::Interface::do_expose"); std::stringstream message; std::string devstr; @@ -2792,8 +2797,6 @@ namespace AstroCam { } } - this->nexp.store(nexp_in); - // Different controllers may have different rules about when to set and clear their // exposure_pending flag (depending on whether or not they support frame transfer), but // if an exposure is pending for any controller then do not start a new exposure. @@ -2807,6 +2810,11 @@ namespace AstroCam { return(ERROR); } + // Only now, and only for a new expose command: a refused expose must not + // change how many sub-exposures the running series has left, and a + // continuation must not undo a stop that lowered nexp since it was read. + if ( !continuation ) this->nexp.store(nexp_in); + // When using the internal ARC shutter control you cannot overlap exposures // with readout because it would require sending a command which cannot // be handled during readout. @@ -3602,9 +3610,10 @@ for ( const auto &dev : selectdev ) { return( ERROR ); } - // Don't remap the DMA buffer while frames are still being written from it. + // Don't remap the DMA buffer while frames are still being written from it: + // give a write that is finishing (ms after READY) up to 2 s, then refuse. // - if ( this->any_writes_pending() ) { + if ( !this->wait_writes_done( 2000 ) ) { logwrite( function, "ERROR: cannot remap DMA buffer while frame write is in progress" ); retstring="write_in_progress"; return ERROR; @@ -4427,9 +4436,18 @@ logwrite(function, message.str()); const std::string function("AstroCam::Interface::stop_exposure"); std::stringstream message; - // nothing to do if not exposing - // + // An NEXP>1 series ends after the frame in progress, whatever that frame + // is doing -- integrating, reading out, or about to start -- so no further + // sub-exposure begins. That frame is also cut short below when it is + // integrating with >= 5 s left. + const bool ends_series = ( this->nexp.load() > 1 ); + if ( ends_series ) this->nexp.store(1); if ( ! this->exposure_pending() ) { + if ( ends_series ) { + logwrite( function, "between sub-exposures: the series ends after the next frame" ); + retstring="series_ends"; + return NO_ERROR; + } logwrite( function, "not exposing" ); return NOTHING; } @@ -4440,6 +4458,12 @@ logwrite(function, message.str()); // long remaining_time = this->camera.shutter_timer.get_remaining(); + if ( remaining_time < 5000 && ends_series ) { + logwrite( function, "too late to cut this frame short: the series ends after it" ); + retstring="series_ends"; + return NO_ERROR; + } + if ( remaining_time < 5000 ) { message << "ERROR cannot stop exposure time with less than 5 s exptime remaining" << " (" << remaining_time << ")"; diff --git a/camerad/astrocam.h b/camerad/astrocam.h index 0901b248..f225ab9a 100644 --- a/camerad/astrocam.h +++ b/camerad/astrocam.h @@ -964,6 +964,26 @@ std::vector> fitsinfo; /***** Interface::any_writes_pending ************************************/ + /***** Interface::wait_writes_done **************************************/ + /** + * @brief wait up to timeout_ms for every frame write to finish + * @return true if no writes are pending, false on timeout + * @details READY is published when the last frame has arrived, a few ms + * before its FITS write ends, so a command sent right after READY + * can find a write still in progress. + */ + inline bool wait_writes_done( int timeout_ms ) { + std::unique_lock lock( this->write_lock ); + return this->write_condition.wait_for( lock, std::chrono::milliseconds( timeout_ms ), [this]() { + for ( const auto &pending : this->writes_pending ) { + if ( !pending.empty() ) return false; + } + return true; + } ); + } + /***** Interface::wait_writes_done **************************************/ + + /***** Interface::write_pending *****************************************/ /** * @brief Set or clear the write pending state for a given exposure @@ -1240,7 +1260,7 @@ std::vector> fitsinfo; void set_imagesize(int rowsin, int colsin, int* status); long expose(std::string nexp_in); - long do_expose(int nexp_in); + long do_expose(int nexp_in, bool continuation=false); ///< continuation: the next sub-exposure of a running series double get_live_airmass(); ///< latest airmass from cached tcsd telemetry, or NAN void get_live_provenance( std::string &githash, std::string &buildtime ); ///< build provenance of all running daemons long native(std::string cmdstr); diff --git a/common/message_keys.h b/common/message_keys.h index 0888bc52..de4e029b 100644 --- a/common/message_keys.h +++ b/common/message_keys.h @@ -106,6 +106,7 @@ namespace Key { inline const std::string ATTEMPTS = "attempts"; inline const std::string SEEING = "seeing"; inline const std::string BACKGROUND = "background"; + inline const std::string GOALSHIFT_PENDING = "goalshift_pending"; ///< a requested goal change awaits execution by the guide loop } namespace Slicecamd { diff --git a/sequencerd/sequence.cpp b/sequencerd/sequence.cpp index 338b6077..c42891c9 100644 --- a/sequencerd/sequence.cpp +++ b/sequencerd/sequence.cpp @@ -51,6 +51,25 @@ namespace Sequencer { if (jmessage.contains(Key::Camerad::READY)) { int isready = jmessage[Key::Camerad::READY].get(); this->can_expose.store(isready); + // The first READY after the next target was picked during a readout is + // the end of that readout (camerad reported READY=false when the + // exposure started and publishes in order): open the readout gate. + if ( isready && this->readout_gate_armed ) { + this->readout_gate_armed = false; + this->wait_state_manager.clear( Sequencer::SEQ_WAIT_READOUT ); + } + } + // Readout counting, step 1 of 2 (see the bookkeeping notes in sequence.h): + // the shutter report going from open to closed, in a report that is not + // PAUSED (camerad closes the shutter to pause, and says so), means an + // integration has ended. Open or paused reports disarm. + if ( jmessage.contains(Key::Camerad::SHUTTEROPEN) && jmessage[Key::Camerad::SHUTTEROPEN].is_boolean() ) { + const bool open = jmessage[Key::Camerad::SHUTTEROPEN].get(); + const bool paused = jmessage.contains(Key::Camerad::PAUSED) && jmessage[Key::Camerad::PAUSED].is_boolean() + && jmessage[Key::Camerad::PAUSED].get(); + if ( open || paused ) this->readout_armed = false; + else if ( this->last_shutteropen ) this->readout_armed = true; + this->last_shutteropen = open; } this->camerad_cv.notify_all(); } @@ -178,6 +197,13 @@ namespace Sequencer { int64_t pubtime=0; Common::extract_telemetry_value( jmessage, Key::PUBTIME, pubtime ); + // acamd's goal change still to be made: raised by offsetgoal, cleared once + // the correction computed against the new goal has been sent and the TCS + // has answered it. wait_for_target_offset waits on it before exposing. + const bool has_goalshift = jmessage.contains( Key::Acamd::GOALSHIFT_PENDING ); + bool goalshift = false; + if ( has_goalshift ) Common::extract_telemetry_value( jmessage, Key::Acamd::GOALSHIFT_PENDING, goalshift ); + // Store under the mutex so the writes are visible to any waiter's predicate // when it re-acquires acam_mtx inside acam_cv.wait(). std::lock_guard lock(this->acam_mtx); @@ -185,6 +211,12 @@ namespace Sequencer { if ( has_mode ) { this->is_acam_acquiring.store( acquiring ); } + // recorded per status, so a wait that starts after the change was already + // made still sees that it was raised and then cleared + if ( has_goalshift ) { + if ( goalshift ) this->acam_goalshift_raised_us.store( pubtime ); + else this->acam_goalshift_cleared_us.store( pubtime ); + } this->acam_pubtime.store( pubtime ); this->acam_cv.notify_all(); } @@ -512,10 +544,47 @@ namespace Sequencer { // Set READOUT flag and clear EXPOSE flag when pixels start coming in // ------------------------------------------------------------------ // - if ( seq.arm_readout_flag && statstr.compare( 0, 10, "PIXELCOUNT" )==0 ) { // async message tag PIXELCOUNT - seq.arm_readout_flag = false; - seq.wait_state_manager.set_and_clear( {Sequencer::SEQ_WAIT_READOUT}, // set READOUT - {Sequencer::SEQ_WAIT_EXPOSE} ); // clear EXPOSE + if ( statstr.compare( 0, 10, "PIXELCOUNT" )==0 ) { // async message tag PIXELCOUNT + // How far into its readout this device is: "PIXELCOUNT_: + // " (the first two numbers after the colon are + // used; -1 if they cannot be read). + double readout_fraction = -1.0; + const auto colon = statstr.find( ':' ); + if ( colon != std::string::npos ) { + std::istringstream fields( statstr.substr( colon+1 ) ); + std::string token; + std::vector numbers; + while ( numbers.size() < 2 && fields >> token ) { + try { + std::size_t used = 0; + const double value = std::stod( token, &used ); + if ( used == token.size() ) numbers.push_back( value ); + } + catch ( const std::exception & ) { } + } + if ( numbers.size() == 2 && numbers[1] > 0.0 ) readout_fraction = numbers[0] / numbers[1]; + } + // The first PIXELCOUNT of a readout ends the EXPOSE wait -- but not one + // from the second half of a readout: that is the tail of the previous + // target's readout, arriving after the next expose command. + if ( seq.arm_readout_flag && !( readout_fraction >= 0.5 ) ) { + seq.arm_readout_flag = false; + seq.wait_state_manager.set_and_clear( {Sequencer::SEQ_WAIT_READOUT}, // set READOUT + {Sequencer::SEQ_WAIT_EXPOSE} ); // clear EXPOSE + } + // Readout counting, step 2 of 2 (step 1 in handletopic_camerad): an + // armed count is completed by a PIXELCOUNT from the first half of a + // readout. A message from the tail of an earlier readout, or one that + // cannot be read, completes nothing; a count completed here leaves at + // least half a readout before camerad can open the shutter again. + { + std::lock_guard lock(seq.camerad_mtx); + if ( seq.readout_armed && readout_fraction >= 0.0 && readout_fraction < 0.5 ) { + seq.readout_armed = false; + seq.readouts_started++; + seq.camerad_cv.notify_all(); + } + } } // --------------------- @@ -640,6 +709,16 @@ namespace Sequencer { this->is_ontarget.store(false); this->is_usercontinue.store(false); + // On every way out of this function except a cancel, do not report the + // sequence finished while camerad is still reading out a target that was + // completed at the start of its last readout. Declared after the + // ScopedStates above, so it runs before they set READY. + // + struct ReadoutGateOnExit { + Sequence &seq; + ~ReadoutGateOnExit() { try { seq.wait_for_readout_gate( "ending the sequence" ); } catch (...) { } } + } readout_gate_on_exit{ *this }; + // This is the main loop which runs as long as there are targets in the database, // or until cancelled. Or, this will run once in the case of single-target-mode, // which is invoked by storing an OBSID in the class variable single_obsid. @@ -696,6 +775,23 @@ namespace Sequencer { } this->publish_targetinfo(); // publish the now-active target + + // A target picked while the previous one is still reading out starts + // at once only if the telescope has to slew to it: a science row with + // coordinates and SLIT pointing. Anything else -- a calibration, a row + // without coordinates, ACAM pointing -- starts when that readout has + // finished, exactly as before. It is already 'active' while it waits, + // like every picked row. + // + { + const bool slews = !this->target.iscal + && !( this->target.ra_hms.empty() && this->target.dec_dms.empty() ) + && !( this->target.pointmode == Acam::POINTMODE_ACAM ); + if ( !slews && !this->wait_for_readout_gate( "starting "+std::string(this->target.name) ) ) { + this->broadcast.notice( function, "sequence cancelled" ); + return; + } + } } else // targetstate not TARGET_FOUND if ( targetstate == TargetInfo::TARGET_NOT_FOUND ) { // no target found is an automatic stop @@ -717,6 +813,14 @@ namespace Sequencer { this->is_repeat_target.store( this->repeat_target() ); if ( this->is_repeat_target.load() ) this->broadcast.notice( function, "repeat target" ); + // Flexure compensation is computed from the telescope position at the + // moment flexured applies it. When the telescope stays where it is (a + // calibration, or a repeat of the same pointing) it is set with the other + // workers, in parallel, as it always was. When the telescope slews, that + // would compensate for the PREVIOUS pointing, so it is set after + // acquisition instead, just before the exposure. + const bool flexure_after_acquisition = !this->target.iscal && !this->is_repeat_target.load(); + // threads to start, pair their ThreadStatusBit with the function to call std::vector>> worker_threads; @@ -739,15 +843,18 @@ namespace Sequencer { // threads to start, pair their ThreadStatusBit with the function to call // + // camera_set is not here: it talks to camerad, so it runs just before + // the exposure, behind the readout gate (see "starting exposure"). worker_threads = { { THR_MOVE_TO_TARGET, std::bind(&Sequence::move_to_target, this) }, - { THR_CAMERA_SET, std::bind(&Sequence::camera_set, this) }, { THR_FOCUS_SET, std::bind(&Sequence::focus_set, this) }, - { THR_FLEXURE_SET, std::bind(&Sequence::flexure_set, this) }, { THR_CALIB_SET, std::bind(&Sequence::calib_set, this) }, // for CAL targets, slit comes from database, otherwise use VSM acquire position { THR_SLIT_SET, std::bind(&Sequence::slit_set, this, this->target.iscal ? Sequencer::VSM_EXPOSE : Sequencer::VSM_ACQUIRE) } }; + if ( !flexure_after_acquisition ) { // telescope does not move: set it now, in parallel + worker_threads.push_back( { THR_FLEXURE_SET, std::bind(&Sequence::flexure_set, this) } ); + } } // pair their ThreadStatusBit with their future @@ -759,6 +866,19 @@ namespace Sequencer { worker_futures.emplace_back( thr, std::async(std::launch::async, func) ); } + // camera_set talks to camerad, so it waits for the readout gate, then runs + // in parallel with everything else (a channel power-up takes seconds) and + // is joined just before the exposure. Not for ACAM pointing, which never + // exposes here. + // + std::future camera_future; + if ( !( this->target.pointmode == Acam::POINTMODE_ACAM ) ) { + camera_future = std::async( std::launch::async, [this]() -> long { + if ( !this->wait_for_readout_gate( "setting the camera for "+std::string(this->target.name) ) ) return ABORT; + return this->camera_set(); + } ); + } + // wait for the threads to complete. these can be cancelled. // for ( auto &[thr, future] : worker_futures) { @@ -830,16 +950,28 @@ namespace Sequencer { } } + // For a target the telescope slewed to (flexure_after_acquisition), the + // flexure compensation is set here -- telescope on target, acquisition + // and target offset done -- in parallel with the slit move and the + // offset settle, and joined before the expose command. For NEXP>1 it is + // re-applied during each readout (flexure_reapply, in the readout wait). + // + std::future flexure_future; + int64_t offset_sent_us = 0; // set when the target offset went to acamd as a goal change + if ( !this->target.iscal ) { // send offsets only on fineacquire, otherwise user needs to fix things if ( !this->is_repeat_target.load() && this->is_fineacquire_locked.load() && - this->target_offset() == ERROR ) { + this->target_offset( &offset_sent_us ) == ERROR ) { if (this->wait_for_user()==ABORT) { this->broadcast.notice( function, "cancelled" ); return; } } + if ( flexure_after_acquisition ) { + flexure_future = std::async( std::launch::async, &Sequence::flexure_set, this ); + } // ensure slit offset is in "expose" position when needed try { error |= this->slit_set(Sequencer::VSM_EXPOSE); @@ -850,9 +982,25 @@ namespace Sequencer { } } - // The offset-star offset is applied asynchronously by the ACAM guide loop; - // ACAMD_OFFSETGOAL returns before the move lands. Wait so the target is on - // the slit before exposing. Skipped when the target has no offset. + // A target offset sent while ACAM guides only changes acamd's goal. The + // guide loop makes the move with its next correction, whose pt_offset + // returns when the TCS answers the move. Expose once acamd reports that + // move made; the wait overlaps the slit move above and is bounded. If the + // move is not reported in time, expose anyway with a warning -- what the + // fixed settle always did -- so this can never hold the sequence. + if ( offset_sent_us > 0 ) { + const long offset_made = this->wait_for_target_offset( offset_sent_us ); + if ( offset_made == ABORT ) { + this->broadcast.notice( function, "cancelled" ); + return; + } + if ( offset_made != NO_ERROR ) { + this->broadcast.warning( function, "target offset not confirmed by acamd in time: exposing anyway" ); + } + } + + // Optional extra settle once the offset has been made (OFFSET_SETTLE_SEC, + // 0 = none). Skipped when the target has no offset. if ( !this->target.iscal && this->is_fineacquire_locked.load() && ( this->target.offset_ra != 0.0 || this->target.offset_dec != 0.0 ) && this->offset_settle_sec > 0.0 ) { @@ -861,6 +1009,53 @@ namespace Sequencer { std::this_thread::sleep_for( std::chrono::duration( this->offset_settle_sec ) ); } + // The compensator must be set before the shutter opens. A failure keeps + // the semantics flexure_set has in the worker set: the sequence stops. + // + if ( flexure_future.valid() ) { + try { + long flexure_error = flexure_future.get(); + error |= flexure_error; + if ( flexure_error != NO_ERROR ) { + this->broadcast.error( function, Sequencer::thread_names.at(THR_FLEXURE_SET)+" failed" ); + } + } + catch (const std::exception& e) { + this->broadcast.error( function, Sequencer::thread_names.at(THR_FLEXURE_SET)+" : "+std::string(e.what()) ); + return; + } + } + + // Nothing reaches camerad before the previous target's readout has + // finished: when this target was picked at the start of that readout, + // the camera settings (camera_future) and the expose wait for READY. + // + if ( !this->wait_for_readout_gate( "exposing "+std::string(this->target.name) ) ) { + this->broadcast.notice( function, "sequence cancelled" ); + return; + } + if ( camera_future.valid() ) { + try { + long camera_error = camera_future.get(); + error |= camera_error; + // never expose unless this target's camera settings were applied + if ( camera_error != NO_ERROR ) { + this->broadcast.error( function, Sequencer::thread_names.at(THR_CAMERA_SET) + +( camera_error==ABORT ? " cancelled" : " failed" )+": not exposing" ); + return; + } + this->broadcast.notice( function, Sequencer::thread_names.at(THR_CAMERA_SET)+" ready" ); + } + catch (const std::exception& e) { + this->broadcast.error( function, Sequencer::thread_names.at(THR_CAMERA_SET)+" : "+std::string(e.what()) ); + return; + } + } + if ( this->cancel_flag.load() ) { + this->broadcast.notice( function, "sequence cancelled" ); + return; + } + logwrite( function, "starting exposure" ); ///< TODO @todo log to telemetry! // Start the exposure in a thread... @@ -889,6 +1084,7 @@ namespace Sequencer { // if ( this->cancel_flag.load() ) { this->broadcast.notice( function, "exposure cancelled" ); + this->arm_readout_flag = false; // the stopped frame's PIXELCOUNTs must not set READOUT later error = this->target.update_state( Sequencer::TARGET_UNASSIGNED ); message.str(""); message << ( error==NO_ERROR ? "" : "ERROR " ) << "marking target " << this->target.name << " id " << this->target.obsid << " order " << this->target.obsorder @@ -907,19 +1103,62 @@ namespace Sequencer { // of the last exposure finishes — the correct completion signal for both // single and multi-exposure sequences. Skip the wait for frame transfer. // + // While waiting, every counted sub-exposure readout (readouts_started; see + // the bookkeeping notes in sequence.h) is acted on once: a readout that more sub-exposures + // follow gets the flexure compensation re-applied during it. When the + // LAST one starts reading out, this target is complete -- its + // integration has ended, and camerad took its header telemetry at + // exposure start and AIRMASS1/MJD1 at shutter close -- so the loop + // completes it and picks the next target now instead of after READY, + // unless single-step, a single target, a cancel or a thread error says + // otherwise. The readout gate keeps camerad untouched until READY. + // + bool picked_during_readout = false; if (!this->is_science_frame_transfer) { logwrite( function, "waiting for readout" ); + int readouts_handled = 0; std::unique_lock lock(this->camerad_mtx); - while ( !this->camerad_cv.wait_for( lock, std::chrono::seconds(30), - [this]() { return this->can_expose.load() || this->cancel_flag.load(); } ) ) { - logwrite( function, "waiting for readout — requesting snapshot" ); + while ( true ) { + const bool signalled = this->camerad_cv.wait_for( lock, std::chrono::seconds(30), + [this,&readouts_handled]() { return this->can_expose.load() || this->cancel_flag.load() || + this->readouts_started > readouts_handled; } ); + if ( this->can_expose.load() || this->cancel_flag.load() ) break; + if ( !signalled ) { + logwrite( function, "waiting for readout — requesting snapshot" ); + lock.unlock(); + this->request_snapshot(); + lock.lock(); + continue; + } + const int sub = ++readouts_handled; // readout of sub-exposure number sub (1-based) has begun + const int expected = this->readouts_expected; // the N sent to camerad, not the live row + if ( sub == expected && !this->target.iscal && !this->do_once.load() && this->single_obsid.empty() && + !this->cancel_flag.load() && !this->thread_error_manager.are_any_set() ) { + this->readout_gate_armed = true; // under camerad_mtx: no READY can slip in between + picked_during_readout = true; + break; + } lock.unlock(); - this->request_snapshot(); + if ( sub < expected ) { + this->flexure_reapply( sub, expected ); + } + else if ( sub > expected ) { + message.str(""); message << "readout " << sub << " seen for a " << expected + << "-exposure command: camerad is running more sub-exposures than were sent"; + this->broadcast.warning( function, message.str() ); + } lock.lock(); } - this->broadcast.notice( function, "done waiting for readout" ); + if ( picked_during_readout ) { + this->broadcast.notice( function, "last readout of "+std::string(this->target.name) + +" under way: completing it and picking the next target" ); + } + else { + this->broadcast.notice( function, "done waiting for readout" ); + } } - this->wait_state_manager.clear( Sequencer::SEQ_WAIT_READOUT ); + // after a pick the READOUT bit stays set until camerad's READY opens the gate + if ( !picked_during_readout ) this->wait_state_manager.clear( Sequencer::SEQ_WAIT_READOUT ); // Now that we're done waiting, check for errors or abort // @@ -996,9 +1235,13 @@ namespace Sequencer { if (this->cancel_flag.load()) { logwrite(function, "sequence cancelled"); - return NO_ERROR; + return ABORT; // nothing was set: the caller must not expose } } + // camerad_mtx guards can_expose only; holding it across the camerad + // commands below would block the ZMQ subscriber thread (every topic) for + // their duration -- a channel power-up takes seconds + lock.unlock(); logwrite( function, "setting camera parameters"); @@ -2149,6 +2392,129 @@ namespace Sequencer { /***** Sequencer::Sequence::camera_shutdown *********************************/ + /***** Sequencer::Sequence::dothread_auto_ontarget **************************/ + /** + * @brief satisfies the TCSOP wait when the TCS itself reports arrival + * @details Spawned by move_to_target when TCS_AUTO_ONTARGET is enabled and + * lives only while that wait is open (auto_ontarget_active). + * Acknowledges when two consecutive 1 Hz reads show + * ?ONTARGET==1 AND motion tracking AND |telescope-commanded| < + * tcs_auto_ontarget_sep AND |TCS cass angle - commanded ring + * angle| < tcs_auto_ontarget_cass. All four are required: the + * flag alone is position-agnostic, the separation alone cannot + * see a sub-threshold hop in progress, motion alone says + * nothing about settling, and none of them covers the cass + * ring, which turns on its own after RINGGO (ACAM's goal is + * computed from the cass angle at the acquire, ~8.5" per + * degree of ring still to go). The coords and ring angle are + * what move_to_target actually commanded. A wrong + * acknowledgment is self-limiting -- ACAM cannot solve a field + * that is not there and the acquisition retry machinery + * handles the failure -- so the conditions make wrong acks + * rare, not impossible. The operator's press works unchanged; + * whichever arrives first wins. The acknowledgment + * deliberately does not go through ontarget(): that clears + * cancel_flag, and a concurrent cancel must always win + * against this automatic path. + */ + void Sequence::dothread_auto_ontarget( double tgt_ra_deg, double tgt_dec_deg, double tgt_cass_deg, unsigned gen ) { + const std::string function("Sequencer::Sequence::dothread_auto_ontarget"); + std::stringstream message; + + if ( std::isnan(tgt_ra_deg) || std::isnan(tgt_dec_deg) || std::isnan(tgt_cass_deg) ) { + logwrite( function, "ERROR invalid commanded coords or ring angle; auto-ontarget sits this wait out" ); + return; + } + + int consecutive = 0; + int64_t ring_wait_since_us = 0; // when everything but the ring first held + bool ring_warned = false; + + // gen retires this watcher the moment a newer one is launched, so one + // still sleeping from an earlier move can never acknowledge a new one. + // + while ( this->auto_ontarget_active.load() && this->auto_ontarget_gen.load() == gen && + !this->cancel_flag.load() && !this->is_ontarget.load() ) { + + std::this_thread::sleep_for( std::chrono::seconds(1) ); + + // the TCS's own settled flag (0|1), returned verbatim + // + std::string reply; + if ( this->tcsd.send( TCSD_NATIVE+" ?ONTARGET", reply ) != NO_ERROR ) { consecutive=0; continue; } + bool flag_on = ( reply.rfind( "1", 0 ) == 0 ); + + // motion state and separation from the cached 1 Hz tcsd telemetry + // + bool fresh = false; + bool tracking = false; + double sep_as = NAN; + double dcass_deg = NAN; + double cass_now = NAN; + { + std::lock_guard lock( tcsd_mtx ); + fresh = this->tcsinfo.is_fresh(); + if ( fresh ) { + tracking = ( this->tcsinfo.motion == TCS_MOTION_TRACKING_STR ); + double dra_deg = this->tcsinfo.ra_h * 15.0 - tgt_ra_deg; + dra_deg = std::fmod( dra_deg + 540.0, 360.0 ) - 180.0; // shortest arc across RA wrap + double dra_as = dra_deg * cos( tgt_dec_deg * M_PI / 180.0 ) * 3600.0; + double ddec_as = ( this->tcsinfo.dec_d - tgt_dec_deg ) * 3600.0; + sep_as = std::hypot( dra_as, ddec_as ); + // the cass ring against the angle RINGGO was sent, either side of the wrap + cass_now = this->tcsinfo.cassangle; + dcass_deg = std::fabs( std::remainder( cass_now - tgt_cass_deg, 360.0 ) ); + } + } + + const bool all_but_ring = fresh && flag_on && tracking && ( sep_as < this->tcs_auto_ontarget_sep ); + const bool ring_ok = ( dcass_deg < this->tcs_auto_ontarget_cass ); + + // On target with only the ring missing for 20 s (a refused RINGGO, a ring + // parked off the command, an unreadable angle): say so once, so the + // operator knows the wait needs a press. + if ( all_but_ring && !ring_ok ) { + if ( ring_wait_since_us == 0 ) ring_wait_since_us = get_time_us(); + else if ( !ring_warned && get_time_us() - ring_wait_since_us > 20000000 ) { + ring_warned = true; + message.str(""); message << "telescope on target but the cass ring reads " << cass_now + << " deg, commanded " << tgt_cass_deg + << " deg: press ontarget once the ring is in place"; + this->broadcast.warning( function, message.str() ); + } + } + else if ( !all_but_ring ) ring_wait_since_us = 0; + + if ( !all_but_ring || !ring_ok ) { consecutive=0; continue; } + + if ( ++consecutive < 2 ) continue; // debounce: two reads in a row + + // Re-check at the last instant and acknowledge under ontarget_ack_mtx, so + // a new move (which retires watchers and clears is_ontarget under the + // same lock) cannot fall between the check and the store. Do NOT clear + // cancel_flag (see @details). cv_mutex is taken before the notify so a + // waiter between its predicate check and its wait cannot miss it. + // + { + std::lock_guard lock( this->ontarget_ack_mtx ); + if ( !this->auto_ontarget_active.load() || this->auto_ontarget_gen.load() != gen || + this->cancel_flag.load() ) return; + this->is_ontarget.store( true ); + } + { std::lock_guard lock( this->cv_mutex ); } + this->cv.notify_all(); + + message.str(""); message << "TCS reports on-target (?ONTARGET=1, tracking, sep=" + << std::fixed << std::setprecision(1) << sep_as + << " arcsec, ring within " << std::setprecision(3) << dcass_deg + << " deg x2) -- sending ontarget"; + this->broadcast.notice( function, message.str() ); + return; + } + } + /***** Sequencer::Sequence::dothread_auto_ontarget **************************/ + + /***** Sequencer::Sequence::move_to_target **********************************/ /** * @brief send request to TCS to move to target coordinates @@ -2186,6 +2552,7 @@ namespace Sequencer { return NO_ERROR; } + error = NO_ERROR; // Before sending target coordinates to TCS, @@ -2260,6 +2627,16 @@ namespace Sequencer { decimal_to_sexa( dec_out, decstr ); message.str(""); message << "[DEBUG] moving to SCOPE COORDS= " << rastr << " " << decstr << " " << angle_out << " J2000"; logwrite( function, message.str() ); + // Retire every auto-ontarget watcher still alive (one from an earlier + // move) and clear the flag, in one step with respect to + // a watcher's acknowledgment: only an ack of THIS move may satisfy the + // wait below. + unsigned ontarget_gen = 0; + { + std::lock_guard lock( this->ontarget_ack_mtx ); + ontarget_gen = ++this->auto_ontarget_gen; + this->is_ontarget.store(false); + } error = this->tcsd.send( coords_cmd.str(), coords_reply ); // send to the TCS // if not success then wait 1s and try again @@ -2297,10 +2674,25 @@ namespace Sequencer { this->broadcast.notice( function, "waiting for TCS operator to send \"ontarget\" signal" ); + // AUTO-ONTARGET (config TCS_AUTO_ONTARGET, default yes): the TCS itself can + // satisfy this wait through the same ontarget() the operator's press uses. + // The watcher lives exactly as long as this wait (auto_ontarget_active). + // + this->auto_ontarget_active.store( this->tcs_auto_ontarget ); + if ( this->tcs_auto_ontarget ) { + // ra_out/dec_out are the SCOPE-frame coordinates this function actually + // commanded, which is the position REQPOS telemetry will report -- the + // database coordinates differ by the pointmode focal-plane offset. + // ontarget_gen was taken, retiring older watchers, before the COORDS. + std::thread( &Sequencer::Sequence::dothread_auto_ontarget, this, + ra_out, dec_out, angle_out, ontarget_gen ).detach(); + } + while ( !this->cancel_flag.load() && !this->is_ontarget.load() ) { std::unique_lock lock(cv_mutex); this->cv.wait( lock, [this]() { return( this->is_ontarget.load() || this->cancel_flag.load() ); } ); } + this->auto_ontarget_active.store( false ); this->broadcast.notice( function, "received " +(this->cancel_flag.load() ? std::string("cancel") : std::string("ontarget")) @@ -2324,6 +2716,41 @@ namespace Sequencer { /***** Sequencer::Sequence::move_to_target **********************************/ + /***** Sequencer::Sequence::wait_for_readout_gate ***************************/ + /** + * @brief wait until the previous target's readout has finished + * @param[in] before what is waiting, for the log + * @return true to proceed, false if cancelled while waiting + * + * The gate is armed when the next target is picked at the start of the + * previous target's last readout (sequence_start) and opened by the first + * camerad READY report after that (handletopic_camerad). Returns at once + * when it is not armed, which is every case but that one. + */ + bool Sequence::wait_for_readout_gate( const std::string &before ) { + const std::string function("Sequencer::Sequence::wait_for_readout_gate"); + std::unique_lock lock( this->camerad_mtx ); + if ( !this->readout_gate_armed ) return true; + logwrite( function, "previous target still reading out: waiting before "+before ); + while ( this->readout_gate_armed && !this->cancel_flag.load() ) { + if ( !this->camerad_cv.wait_for( lock, std::chrono::seconds(30), + [this]() { return !this->readout_gate_armed || this->cancel_flag.load(); } ) ) { + logwrite( function, "still waiting for the previous readout -- requesting snapshot" ); + lock.unlock(); + this->request_snapshot(); + lock.lock(); + } + } + if ( this->readout_gate_armed ) { + logwrite( function, "cancelled while waiting for the previous readout" ); + return false; + } + logwrite( function, "previous readout finished" ); + return true; + } + /***** Sequencer::Sequence::wait_for_readout_gate ***************************/ + + /***** Sequencer::Sequence::dothread_notify_tcs *****************************/ /** * @brief notify TCS operator of next target coords for preauthorization, no move @@ -2452,31 +2879,110 @@ namespace Sequencer { return NO_ERROR; } - // build up list of activate chans + // the compensate command names this target's active channels + const std::string comp = this->flexure_compensate_command(); + const std::string activechans = comp.substr( std::string(FLEXURED_COMPENSATE).size() ); + + if ( !this->cancel_flag.load() && + this->flexured.command( comp ) != NO_ERROR ) { + this->broadcast.error( function, "setting flexure compensator" ); + this->thread_error_manager.set( THR_FLEXURE_SET ); + throw std::runtime_error("setting flexure compensator"); + } + + this->is_flexure_default.store(false); + + this->broadcast.notice( function, "flexure compensator set for "+activechans); + + return NO_ERROR; + } + /***** Sequencer::Sequence::flexure_set *************************************/ + + + /***** Sequencer::Sequence::flexure_compensate_command **********************/ + /** + * @brief the FLEXURED_COMPENSATE command for the current target + * @return "compensate [ ...]" with the channels active for + * the target's calibration entry ("SCIENCE" for science targets) + */ + std::string Sequence::flexure_compensate_command() { std::ostringstream activechans; const std::string calname = std::string(this->target.iscal ? this->target.name : "SCIENCE"); const auto &calinfo = this->caltarget.get_info(calname); for (const auto &[chan,active] : calinfo.channel_active) { if (active) activechans << " " << chan; } + return std::string(FLEXURED_COMPENSATE) + activechans.str(); + } + /***** Sequencer::Sequence::flexure_compensate_command **********************/ - std::ostringstream comp; - comp << FLEXURED_COMPENSATE << activechans.str(); - if ( !this->cancel_flag.load() && - this->flexured.command( comp.str() ) != NO_ERROR ) { - this->broadcast.error( function, "setting flexure compensator" ); - this->thread_error_manager.set( THR_FLEXURE_SET ); - throw std::runtime_error("setting flexure compensator"); + /***** Sequencer::Sequence::flexure_reapply *********************************/ + /** + * @brief re-issue the flexure compensation between two sub-exposures + * @param[in] sub the sub-exposure (1-based) whose readout is in progress + * @return ERROR | NO_ERROR (informational only) + * + * Called by sequence_start during the readout of sub-exposure sub when + * sub-exposure sub+1 of expected follows. flexured recomputes from the + * telescope position at this moment, so the next sub-exposure starts + * corrected. Skipped (the compensation set at the start of the series + * stays) unless that readout is still running and the next shutter has not + * opened, the TCS telemetry is fresh and finite (flexured itself does not + * check the values), and no thread error is set. A failure is reported and + * otherwise ignored: nothing may disturb a running exposure command. + */ + long Sequence::flexure_reapply( int sub, int expected ) { + const std::string function("Sequencer::Sequence::flexure_reapply"); + std::stringstream message; + + if ( !this->should_flexcomp.load() || this->cancel_flag.load() ) return NO_ERROR; + + auto skipped = [&]( const std::string &why ) { + message.str(""); message << "flexure compensation NOT re-applied before exposure " << sub+1 + << " of " << expected << ": " << why; + this->broadcast.warning( function, message.str() ); + return ERROR; + }; + auto next_started = [this, sub]() { + std::lock_guard lock( this->camerad_mtx ); + return ( this->last_shutteropen || this->readouts_started != sub ); + }; + + if ( this->thread_error_manager.are_any_set() ) return skipped( "a thread error is set" ); + { + std::lock_guard lock( this->tcsd_mtx ); + if ( !this->daemon_manager.is_set( Sequencer::DAEMON_TCS ) || !this->tcsinfo.is_fresh() || + !std::isfinite( this->tcsinfo.zenithangle ) || !std::isfinite( this->tcsinfo.cassangle ) || + !std::isfinite( this->tcsinfo.pa ) ) { + return skipped( "TCS telemetry not fresh or not valid" ); + } } + if ( next_started() ) return skipped( "the next exposure has already started" ); + + const std::string comp = this->flexure_compensate_command(); + + message.str(""); message << "re-applying flexure compensation during readout " << sub << " of " + << expected << " for " << this->target.name; + logwrite( function, message.str() ); + + if ( this->flexured.command( comp ) != NO_ERROR ) return skipped( "flexured returned an error" ); this->is_flexure_default.store(false); - this->broadcast.notice( function, "flexure compensator set for "+activechans.str()); + if ( next_started() ) { + message.str(""); message << "flexure compensator re-applied, but exposure " << sub+1 << " of " << expected + << " had already started: the actuators moved during its first seconds"; + this->broadcast.warning( function, message.str() ); + return ERROR; + } + message.str(""); message << "flexure compensator re-applied before exposure " << sub+1 + << " of " << expected; + this->broadcast.notice( function, message.str() ); return NO_ERROR; } - /***** Sequencer::Sequence::flexure_set *************************************/ + /***** Sequencer::Sequence::flexure_reapply *********************************/ /***** Sequencer::Sequence::calib_set ***************************************/ @@ -2688,7 +3194,9 @@ namespace Sequencer { // so this loop exits after at most a few iterations. // if ( abort_during_run ) { - const auto drain_timeout = std::chrono::steady_clock::now() + std::chrono::seconds(5); + // 20 s: the readout wait may be inside a flexured re-apply (client + // timeout 6 s + 2 s, flexured itself up to ~11 s) when the cancel lands + const auto drain_timeout = std::chrono::steady_clock::now() + std::chrono::seconds(20); while ( this->thread_state_manager.is_set( Sequencer::THR_SEQUENCE_START ) && std::chrono::steady_clock::now() < drain_timeout ) { std::this_thread::sleep_for( std::chrono::milliseconds(20) ); @@ -2732,15 +3240,24 @@ namespace Sequencer { ScopedState thr_state( this->thread_state_manager, Sequencer::THR_STOP_EXPOSURE ); - // This function is only used while exposing + // Only while an exposure command of the current target is running. For + // NEXP>1 that is the whole series: the EXPOSE bit covers only the first + // sub-exposure; after it, camerad not READY (and not the previous + // target's last readout after a pick, which has nothing left to stop) + // covers the rest. A stale READOUT bit is not evidence. camerad ends the + // series after the frame in progress, cut short if it is integrating + // with >= 5 s left. A stop never clears a cancel. // - if ( ! this->wait_state_manager.is_set( Sequencer::SEQ_WAIT_EXPOSE ) ) { + bool series_running = false; + { + std::lock_guard lock( this->camerad_mtx ); + series_running = !this->can_expose.load() && !this->readout_gate_armed; + } + if ( ! this->wait_state_manager.is_set( Sequencer::SEQ_WAIT_EXPOSE ) && !series_running ) { this->broadcast.notice(function, "not currently exposing"); return; } - this->cancel_flag.store(false); - // Send command to the camera to stop the exposure. // std::string reply; @@ -2797,6 +3314,8 @@ namespace Sequencer { this->is_ontarget.store(false); this->is_usercontinue.store(false); + if ( !this->wait_for_readout_gate( "repeating the exposure" ) ) return ERROR; + std::string targetstatus; this->target.get_specified_target( this->prev_single_obsid, targetstatus ); @@ -2840,6 +3359,24 @@ namespace Sequencer { return NO_ERROR; } + // Flexure for the telescope position now. Only after camera_set has seen + // camerad ready: a REPEAT is accepted while the previous one is still + // integrating, and the actuators must not move under it. + // + try { + if ( this->flexure_set() != NO_ERROR ) { + this->broadcast.error( function, Sequencer::thread_names.at(THR_FLEXURE_SET)+" failed" ); + } + } + catch (const std::exception& e) { + this->broadcast.error( function, Sequencer::thread_names.at(THR_FLEXURE_SET)+" exception: "+std::string(e.what()) ); + return ERROR; + } + if ( this->cancel_flag.load() ) { + this->broadcast.notice(function, "cancelled repeat exposure"); + return NO_ERROR; + } + // Start the exposure in a thread... // auto start_exposure = std::async(std::launch::async, &Sequence::trigger_exposure, this); @@ -2890,6 +3427,13 @@ namespace Sequencer { // if ( this->tcs_preauth_time > 0 ) this->notify_tcs_next_target = true; else this->notify_tcs_next_target = false; + { + std::lock_guard lock(this->camerad_mtx); + this->last_shutteropen = false; // readout bookkeeping starts fresh with this expose command; + this->readout_armed = false; // the shutter is closed until camerad opens it for this one + this->readouts_started = 0; + this->readouts_expected = this->target.nexp; // the N sent below, fixed for this command + } this->arm_readout_flag = true; // enables the async_listener to look for the readout and clear the EXPOSE bit this->set_imgtype(); @@ -3840,9 +4384,11 @@ namespace Sequencer { * @return ERROR or NO_ERROR * */ - long Sequence::target_offset() { + long Sequence::target_offset( int64_t *acam_sent_us ) { const std::string function("Sequencer::Sequence::target_offset"); + if ( acam_sent_us ) *acam_sent_us = 0; + // nothing to do if both ra and dec offsets are zero if (this->target.offset_ra == 0.0 && this->target.offset_dec == 0.0) return NO_ERROR; @@ -3857,7 +4403,11 @@ namespace Sequencer { const double ddec_deg = this->target.offset_dec / 3600.0; std::stringstream cmd; cmd << ACAMD_OFFSETGOAL << " " << std::fixed << std::setprecision(6) << dra_deg << " " << ddec_deg; + // taken before the send, so acamd's publish of the raised goal change is + // always later (no guard band needed, unlike do_acam_acquire) + const int64_t sent_us = get_time_us(); error = this->acamd.command( cmd.str() ); + if ( error == NO_ERROR && acam_sent_us ) *acam_sent_us = sent_us; } else // if ACAM is not guiding then I send the offsets directly to the TCS @@ -3876,6 +4426,61 @@ namespace Sequencer { /***** Sequencer::Sequence::target_offset ***********************************/ + /***** Sequencer::Sequence::wait_for_target_offset **************************/ + /** + * @brief wait until acamd has made a target offset sent as a goal change + * @details ACAMD_OFFSETGOAL only changes the guide goal. acamd's guide loop + * makes the move with its next correction, and that correction's + * pt_offset returns when the TCS answers the move (later for + * bigger moves). acamd raises GOALSHIFT_PENDING with the goal + * change and clears it once that correction has been sent and + * answered. The move is made once a status published after the + * send has shown it raised and a later one has shown it cleared. + * handletopic_acamd records both publish times as statuses + * arrive, so this holds even if the move was made before this + * wait began (the wait follows the slit move). Abort wakes acam_cv. + * @param[in] sent_us get_time_us() taken just before the offset was sent + * @return NO_ERROR | ERROR (not reported within TIMEOUT_SEC) | ABORT (cancelled) + * + */ + long Sequence::wait_for_target_offset( const int64_t sent_us ) { + const std::string function("Sequencer::Sequence::wait_for_target_offset"); + + // The move is made at acamd's next ACAM frame and answered by the TCS + // within seconds (measured: <= 4.3 s up to 147"); this covers a second + // frame and a 300" move. Bounded: the caller exposes either way. + // + constexpr double TIMEOUT_SEC = 30.0; + + logwrite( function, "NOTICE: waiting for acamd to make the target offset before exposing" ); + + const auto deadline = std::chrono::steady_clock::now() + std::chrono::duration( TIMEOUT_SEC ); + bool made = false; + { + std::unique_lock lock( this->acam_mtx ); + made = this->acam_cv.wait_until( lock, deadline, [&]() { + if ( this->cancel_flag.load() ) return true; + const int64_t raised = this->acam_goalshift_raised_us.load(); + return ( raised > sent_us && this->acam_goalshift_cleared_us.load() > raised ); + } ); + } + + if ( this->cancel_flag.load() ) return ABORT; + + std::ostringstream oss; + if ( !made ) { + oss << "ERROR acamd did not make the target offset within " << TIMEOUT_SEC << " s"; + logwrite( function, oss.str() ); + return ERROR; + } + oss << "target offset made by acamd " << std::fixed << std::setprecision(1) + << ( get_time_us() - sent_us ) / 1.0e6 << " s after it was sent"; + logwrite( function, oss.str() ); + return NO_ERROR; + } + /***** Sequencer::Sequence::wait_for_target_offset **************************/ + + /***** Sequencer::Sequence::publish_targetinfo *****************************/ /** * @brief publish target info on Topic::TARGETINFO, on change (or force) diff --git a/sequencerd/sequence.h b/sequencerd/sequence.h index 8df8f33b..97328b4f 100644 --- a/sequencerd/sequence.h +++ b/sequencerd/sequence.h @@ -305,6 +305,8 @@ namespace Sequencer { std::atomic is_acam_guiding{false}; ///< is acam guiding (IS_ACQUIRED)? std::atomic is_acam_acquiring{false}; ///< is acam in an acquire mode? std::atomic acam_pubtime{0}; ///< publish time (us) of latest received acamd status + std::atomic acam_goalshift_raised_us{0}; ///< publish time (us) of the latest acamd status showing GOALSHIFT_PENDING set + std::atomic acam_goalshift_cleared_us{0}; ///< publish time (us) of the latest acamd status showing GOALSHIFT_PENDING clear /** @brief guard-band (us) subtracted from the acquire-command send time * when computing the freshness boundary in do_acam_acquire. Tolerates @@ -350,7 +352,7 @@ namespace Sequencer { tcs_settle_stable(1), tcs_domeazi_ready(1), tcs_preauth_time(0), - offset_settle_sec(3), + offset_settle_sec(0), do_once(false), tcs_which("real"), tcs_name("offline"), @@ -399,6 +401,14 @@ namespace Sequencer { this->is_ontarget.store(false); } + /** @brief satisfies the TCSOP wait when the TCS itself reports arrival + * (config TCS_AUTO_ONTARGET); coords are the commanded SCOPE + * frame position in degrees; see sequence.cpp */ + void dothread_auto_ontarget( double tgt_ra_deg, double tgt_dec_deg, double tgt_cass_deg, unsigned gen ); + std::atomic auto_ontarget_active{false}; ///< true only while move_to_target's TCSOP wait is open + std::atomic auto_ontarget_gen{0}; ///< launch generation; a watcher retires when a newer one is launched + std::mutex ontarget_ack_mtx; ///< a watcher's acknowledgment vs a new move retiring watchers and clearing is_ontarget + /** @brief sends usercontinue signal, clears after 3 seconds */ inline void usercontinue() { this->cancel_flag.store(false); @@ -420,7 +430,10 @@ namespace Sequencer { double tcs_settle_stable; ///< time that TCS must report TRACKING before it is really tracking double tcs_domeazi_ready; ///< max degrees azimuth that dome and telescope can differ before ready to observe double tcs_preauth_time; ///< seconds before end of exposure to notify TCS of next target's coords (0 to disable) - double offset_settle_sec; ///< sec to wait after a target offset before exposing (config OFFSET_SETTLE_SEC; 0 disables) + double offset_settle_sec; ///< extra sec to wait after the target offset has been made, before exposing (config OFFSET_SETTLE_SEC; 0 = none) + bool tcs_auto_ontarget = true; ///< TCS ?ONTARGET may satisfy the TCSOP wait (config TCS_AUTO_ONTARGET; default yes) + double tcs_auto_ontarget_sep = 5.0; ///< max |telescope-target| arcsec for auto-ontarget (config TCS_AUTO_ONTARGET_SEP) + double tcs_auto_ontarget_cass = 0.05; ///< max |TCS cass angle - commanded ring angle| deg for auto-ontarget (config TCS_AUTO_ONTARGET_CASS) /// std::mutex tcs_ontarget_mtx; /// std::condition_variable tcs_ontarget_cv; @@ -431,6 +444,27 @@ namespace Sequencer { std::condition_variable acam_cv; std::mutex camerad_mtx; std::condition_variable camerad_cv; + // Readout bookkeeping for the current expose command, guarded by camerad_mtx + // and reset in trigger_exposure(). A sub-exposure's readout is counted in two + // steps. (1) camerad's shutter report goes from open to closed in a report + // that is not PAUSED: that integration has ended, so the count is armed + // (handletopic_camerad). (2) A PIXELCOUNT from the first half of a readout + // completes it (async listener). Any open or paused report disarms. + // camerad's reports arrive in the order it published them, so a late or + // repeated report cannot arm twice for one close, and a late PIXELCOUNT + // cannot count a sub-exposure that is integrating. The count never exceeds + // the integrations that have ended; a lost report only leaves one uncounted. + bool last_shutteropen = false; ///< SHUTTEROPEN in the last camerad report since the expose command + bool readout_armed = false; ///< an integration ended (open -> closed, not paused), its readout not yet counted + int readouts_started = 0; ///< sub-exposure readouts counted since the expose command + int readouts_expected = 0; ///< N of that expose command (this->target can be rewritten by manual commands) + // The readout gate, guarded by camerad_mtx. When the last readout of a + // target starts, the loop completes that target and picks the next one + // at once; nothing may then reach camerad (camera settings, an expose) + // until camerad reports READY again. Armed at that pick, opened by the + // first READY=true report after it (handletopic_camerad), and held + // across a stop or restart until then. + bool readout_gate_armed = false; std::mutex slitd_mtx; std::condition_variable slitd_cv; std::mutex tcsd_mtx; @@ -589,7 +623,8 @@ namespace Sequencer { long get_tcs_weather_coords( double &ra_h, double &dec_d ); ///< read the current TCS ra,dec in decimal hr,deg long get_tcs_coords( double &ra_h, double &dec_d ); ///< read the current TCS ra,dec,cass in decimal hr,deg long get_tcs_cass( double &cass ); - long target_offset(); + long target_offset( int64_t *acam_sent_us = nullptr ); ///< acam_sent_us: send time if the offset went to acamd as a goal change, else 0 + long wait_for_target_offset( const int64_t sent_us ); ///< wait until acamd has made that goal change void publish_targetinfo( bool force=false ); ///< publish target info on change (or force) @@ -619,6 +654,9 @@ namespace Sequencer { long camera_set(); ///< sets camera according to target entry params long slit_set(VirtualSlitMode mode=VSM_DATABASE); ///< sets slit according to target entry params and mode long move_to_target(); ///< sends request to TCS to move to target coords + bool wait_for_readout_gate( const std::string &before ); ///< wait for the previous target's readout to finish; false if cancelled + long flexure_reapply( int sub, int expected ); ///< re-issues the flexure compensation between sub-exposures (never fatal) + std::string flexure_compensate_command(); ///< FLEXURED_COMPENSATE plus the active channels for the current target static void dothread_notify_tcs( Sequencer::Sequence &seq ); ///< like move_to_target but for preauth only long focus_set(); long flexure_set(); diff --git a/sequencerd/sequencer_server.cpp b/sequencerd/sequencer_server.cpp index 405b43f3..9b37a371 100644 --- a/sequencerd/sequencer_server.cpp +++ b/sequencerd/sequencer_server.cpp @@ -362,6 +362,46 @@ namespace Sequencer { applied++; } + // TCS_AUTO_ONTARGET + if (config.param[entry] == "TCS_AUTO_ONTARGET") { + this->sequence.tcs_auto_ontarget = ( config.arg[entry] == "yes" || + config.arg[entry] == "true" || + config.arg[entry] == "1" ); + message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; + logwrite( function, message.str() ); + applied++; + } + + // TCS_AUTO_ONTARGET_CASS + if (config.param[entry] == "TCS_AUTO_ONTARGET_CASS") { + try { + this->sequence.tcs_auto_ontarget_cass = std::stod( config.arg[entry] ); + } + catch (const std::exception &e) { + message.str(""); message << "ERROR parsing TCS_AUTO_ONTARGET_CASS: " << e.what(); + logwrite( function, message.str() ); + return ERROR; + } + message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; + logwrite( function, message.str() ); + applied++; + } + + // TCS_AUTO_ONTARGET_SEP + if (config.param[entry] == "TCS_AUTO_ONTARGET_SEP") { + try { + this->sequence.tcs_auto_ontarget_sep = std::stod( config.arg[entry] ); + } + catch (const std::exception &e) { + message.str(""); message << "ERROR parsing TCS_AUTO_ONTARGET_SEP: " << e.what(); + logwrite( function, message.str() ); + return ERROR; + } + message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; + logwrite( function, message.str() ); + applied++; + } + // OFFSET_SETTLE_SEC if (config.param[entry] == "OFFSET_SETTLE_SEC") { try { diff --git a/slicecamd/slicecam_interface.cpp b/slicecamd/slicecam_interface.cpp index ec21083e..0618c7e1 100644 --- a/slicecamd/slicecam_interface.cpp +++ b/slicecamd/slicecam_interface.cpp @@ -60,6 +60,10 @@ namespace Slicecam { const bool was_running = this->is_fineacquire_running.load(std::memory_order_acquire); this->is_fineacquire_locked.store(false, std::memory_order_release); this->is_fineacquire_running.store(false, std::memory_order_release); + { + std::lock_guard lock(this->acam_mtx); + this->acam_cv.notify_all(); + } this->publish_status(); logwrite(function, was_running ? "stop requested" : "stopped"); retstring=this->is_fineacquire_running.load(std::memory_order_acquire)?"running":"stopped"; @@ -165,6 +169,7 @@ namespace Slicecam { this->fineacquire_state.reset(); this->fineacq_total_dra = 0.0; // reset per-run ACAM->slit residual accumulators this->fineacq_total_ddec = 0.0; + this->fineacq_adjustments = 0; // relaxed-accept counter, per run // snapshot this run's goal (DB target) coords from the TARGETINFO published message; NAN if no target // has been published (manual runs log nan). Frozen for the run via the is_fineacquire_running handoff. @@ -419,6 +424,43 @@ namespace Slicecam { const int max_samples = this->fineacquire_state.max_samples; const int min_samples = this->fineacquire_state.min_samples; + // Verification pass, accept-only early exit: once a correction has been + // applied this run, a single frame already inside the goal is sufficient + // proof. A frame may only ACCEPT -- anything larger falls through to the + // usual median machinery, so a noisy frame can never trigger a + // correction or an extra cycle. + // + if ( this->fineacq_total_dra != 0.0 || this->fineacq_total_ddec != 0.0 ) { + const double r1 = std::hypot( offsets.first, offsets.second ) * 3600.0; + if ( r1 <= this->fineacquire_state.goal_arcsec ) { + // the same converged line as every lock (log readers key on it) + std::ostringstream oss; + oss << "fine acquisition converged: offset dRA=" << offsets.first * 3600.0 + << " dDEC=" << offsets.second * 3600.0 + << " arcsec (r=" << r1 + << " arcsec, n=1 scatter=(0,0) arcsec)" + << " goal=" << this->fineacquire_state.goal_arcsec << " arcsec" + << "; accepted on a single frame after a correction"; + logwrite( function, oss.str() ); + + std::ostringstream acqmodel; + acqmodel << "[ACQMODEL] acam2slit dRA=" << this->fineacq_total_dra + << " dDEC=" << this->fineacq_total_ddec << " arcsec" + << " GOALRA=" << this->fineacq_goal_ra + << " GOALDEC=" << this->fineacq_goal_dec + << " CASANGLE=" << this->telem.angle_scope + << " n=" << n + << " cam=" << which; + logwrite( function, acqmodel.str() ); + + this->is_fineacquire_locked.store( true, std::memory_order_release ); + this->is_fineacquire_running.store( false, std::memory_order_release ); + this->fineacquire_state.reset(); + this->publish_status(); + return; + } + } + // wait for the minimum number of samples before evaluating anything // if ( n < min_samples ) return; @@ -541,6 +583,30 @@ namespace Slicecam { } } + // THE BUDGET'S LAST CORRECTION (FINE_ACQUIRE_MAX_ADJUSTMENTS): the run can + // lock after it only if the offset it would remove is within + // FINE_ACQUIRE_RELAXED_FACTOR x goal, and that offset is known now. When it + // is larger the run fails here, without sending a correction that no lock + // can follow (the sequencer reports a failed acquisition; the robot retries). + // + if ( this->fineacquire_state.max_adjustments > 0 + && this->fineacq_adjustments + 1 >= this->fineacquire_state.max_adjustments + && !( offset_arcsec <= this->fineacquire_state.relaxed_factor + * this->fineacquire_state.goal_arcsec ) ) { + std::ostringstream oss; + oss << "fine acquisition FAILED: correction " << this->fineacq_adjustments + 1 + << " of " << this->fineacquire_state.max_adjustments << " would be the last and r=" + << offset_arcsec << " arcsec > relaxed goal=" + << this->fineacquire_state.relaxed_factor * this->fineacquire_state.goal_arcsec + << " arcsec, so no lock can follow; not sent"; + logwrite( function, oss.str() ); + this->is_fineacquire_locked.store( false, std::memory_order_release ); + this->is_fineacquire_running.store( false, std::memory_order_release ); + this->fineacquire_state.reset(); + this->publish_status(); + return; + } + // select gain: use gain_large when offset is well above the goal threshold // const double effective_gain = ( offset_arcsec > this->fineacquire_state.gain_threshold_arcsec ) @@ -562,12 +628,55 @@ namespace Slicecam { const double cmd_dra = effective_gain * med_dra; const double cmd_ddec = effective_gain * med_ddec; + const int64_t send_time = get_time_us(); + const bool was_guiding = this->is_acam_guiding.load(); + if ( this->offset_acam_goal( { cmd_dra, cmd_ddec }, true ) != NO_ERROR ) { logwrite( function, "ERROR failed to send offset to ACAM" ); this->is_fineacquire_running.store( false, std::memory_order_release ); this->publish_status(); return; } + this->fineacq_adjustments++; // one more commanded correction this run (relaxed accept) + + // while guiding the correction only re-points acamd's goal; wait until + // acamd reports the goal change applied before settling + if ( was_guiding && this->fineacquire_state.move_timeout_sec > 0.0 ) { + std::unique_lock lock(this->acam_mtx); + // completion requires a status after the send showing the request + // pending, followed by one showing it cleared + const bool applied = this->acam_cv.wait_for( lock, + std::chrono::duration( this->fineacquire_state.move_timeout_sec ), + [this, send_time]() { + if ( !this->is_fineacquire_running.load(std::memory_order_acquire) || + !this->should_framegrab_run.load(std::memory_order_acquire) ) return true; + const int64_t raised = this->acam_goalshift_raised_us.load(std::memory_order_acquire); + return ( raised > send_time && + this->acam_goalshift_cleared_us.load(std::memory_order_acquire) > raised ); + }); + if ( !this->should_framegrab_run.load(std::memory_order_acquire) ) return; + if ( !this->is_fineacquire_running.load(std::memory_order_acquire) ) return; + if ( !applied ) { + std::ostringstream w; + if ( this->fineacquire_state.max_adjustments > 0 ) { + // A budgeted run decides on the corrections it has commanded, so it + // cannot go on from one that never executed: later frames would + // still show the old position, and the shift may land mid-exposure. + w << "fine acquisition FAILED: correction " << this->fineacq_adjustments << " of " + << this->fineacquire_state.max_adjustments << " was not confirmed applied by acamd within " + << this->fineacquire_state.move_timeout_sec << " s"; + logwrite( function, w.str() ); + this->is_fineacquire_locked.store( false, std::memory_order_release ); + this->is_fineacquire_running.store( false, std::memory_order_release ); + this->fineacquire_state.reset(); + this->publish_status(); + return; + } + w << "WARNING goal change not applied within " + << this->fineacquire_state.move_timeout_sec << " s; proceeding"; + logwrite( function, w.str() ); + } + } // time-based settle: wait for the TCS to physically finish the move before // sampling resumes. The frame-count settle (settle_frames) is too short in @@ -576,11 +685,55 @@ namespace Slicecam { if ( this->fineacquire_state.settle_sec > 0.0 ) { std::this_thread::sleep_for( std::chrono::duration( this->fineacquire_state.settle_sec ) ); } + // a stop that arrived during the settle ends this run here: nothing below + // (totals, the budget's decision, a published lock) may act for it + if ( !this->should_framegrab_run.load(std::memory_order_acquire) || + !this->is_fineacquire_running.load(std::memory_order_acquire) ) return; // accumulate the applied correction (arcsec). Summed over the run this is the // total ACAM->slit residual that acam-acquire left behind (the [ACQMODEL] line). this->fineacq_total_dra += cmd_dra * 3600.0; this->fineacq_total_ddec += cmd_ddec * 3600.0; + // The budget's last correction decides. Once FINE_ACQUIRE_MAX_ADJUSTMENTS + // corrections have been commanded, nothing further is sampled or evaluated. + // This last one was only sent because the offset it removes is within + // FINE_ACQUIRE_RELAXED_FACTOR x goal (checked before the send), and it has + // executed and settled, so the run converges now. 0 disables the budget + // (correct until the goal or the sequencer timeout). + // + if ( this->fineacquire_state.max_adjustments > 0 + && this->fineacq_adjustments >= this->fineacquire_state.max_adjustments ) { + std::ostringstream oss; + // the same converged line as every lock (log readers key on it); the + // offset is the one this last correction was sent for, and the goal the + // relaxed one it was accepted against + oss << "fine acquisition converged: offset dRA=" << med_dra * 3600.0 + << " dDEC=" << med_ddec * 3600.0 + << " arcsec (r=" << offset_arcsec + << " arcsec, n=" << n + << " scatter=(" << sig_dra << "," << sig_ddec << ") arcsec)" + << " goal=" << this->fineacquire_state.relaxed_factor * this->fineacquire_state.goal_arcsec + << " arcsec; accepted on the final correction (" << this->fineacq_adjustments + << " of " << this->fineacquire_state.max_adjustments << ") without a verification frame"; + logwrite( function, oss.str() ); + + std::ostringstream acqmodel; + acqmodel << "[ACQMODEL] acam2slit dRA=" << this->fineacq_total_dra + << " dDEC=" << this->fineacq_total_ddec << " arcsec" + << " GOALRA=" << this->fineacq_goal_ra + << " GOALDEC=" << this->fineacq_goal_dec + << " CASANGLE=" << this->telem.angle_scope + << " n=" << n + << " cam=" << which; + logwrite( function, acqmodel.str() ); + + this->is_fineacquire_locked.store( true, std::memory_order_release ); + this->is_fineacquire_running.store( false, std::memory_order_release ); + this->fineacquire_state.reset(); + this->publish_status(); + return; + } + // reset samples and discard settle_count frames for telescope settling this->fineacquire_state.reset(); this->fineacquire_state.settle_frames = this->fineacquire_state.settle_count; @@ -924,6 +1077,14 @@ namespace Slicecam { Common::extract_telemetry_value( jmessage, Key::PUBTIME, pubtime ); this->last_acam_pubtime.store( pubtime, std::memory_order_relaxed ); + // when acamd last reported a goal change pending, and when it last reported none + if ( jmessage.contains( Key::Acamd::GOALSHIFT_PENDING ) ) { + bool goalshift_pending=false; + Common::extract_telemetry_value( jmessage, Key::Acamd::GOALSHIFT_PENDING, goalshift_pending ); + if ( goalshift_pending ) this->acam_goalshift_raised_us.store( pubtime, std::memory_order_relaxed ); + else this->acam_goalshift_cleared_us.store( pubtime, std::memory_order_relaxed ); + } + // wake any thread waiting on ACAM state (e.g. fineacquire) std::lock_guard lock(this->acam_mtx); this->acam_cv.notify_all(); @@ -1418,6 +1579,53 @@ namespace Slicecam { applied++; } else + if ( config.param[entry] == "FINE_ACQUIRE_MAX_ADJUSTMENTS" ) { + try { this->fineacquire_state.max_adjustments = std::stoi( config.arg[entry] ); } + catch ( const std::exception &e ) { + message.str(""); message << "ERROR invalid FINE_ACQUIRE_MAX_ADJUSTMENTS " + << config.arg[entry] << ": " << e.what(); + logwrite( function, message.str() ); + return ERROR; + } + if ( this->fineacquire_state.max_adjustments < 0 ) { + logwrite( function, "ERROR FINE_ACQUIRE_MAX_ADJUSTMENTS must be >= 0 (0 disables the budget)" ); + return ERROR; + } + message.str(""); message << "SLICECAMD:config:" << config.param[entry] << "=" << config.arg[entry]; + logwrite( function, message.str() ); + applied++; + } + else + if ( config.param[entry] == "FINE_ACQUIRE_RELAXED_FACTOR" ) { + try { this->fineacquire_state.relaxed_factor = std::stod( config.arg[entry] ); } + catch ( const std::exception &e ) { + message.str(""); message << "ERROR invalid FINE_ACQUIRE_RELAXED_FACTOR " + << config.arg[entry] << ": " << e.what(); + logwrite( function, message.str() ); + return ERROR; + } + if ( !( this->fineacquire_state.relaxed_factor >= 1.0 ) ) { // also refuses NaN + logwrite( function, "ERROR FINE_ACQUIRE_RELAXED_FACTOR must be a number >= 1" ); + return ERROR; + } + message.str(""); message << "SLICECAMD:config:" << config.param[entry] << "=" << config.arg[entry]; + logwrite( function, message.str() ); + applied++; + } + else + if ( config.param[entry] == "FINE_ACQUIRE_MOVE_TIMEOUT" ) { + try { this->fineacquire_state.move_timeout_sec = std::stod( config.arg[entry] ); } + catch ( const std::exception &e ) { + message.str(""); message << "ERROR invalid FINE_ACQUIRE_MOVE_TIMEOUT " + << config.arg[entry] << ": " << e.what(); + logwrite( function, message.str() ); + return ERROR; + } + message.str(""); message << "SLICECAMD:config:" << config.param[entry] << "=" << config.arg[entry]; + logwrite( function, message.str() ); + applied++; + } + else if ( config.param[entry] == "FINE_ACQUIRE_GAIN" ) { try { this->fineacquire_state.gain = std::stod( config.arg[entry] ); } catch ( const std::exception &e ) { @@ -1634,6 +1842,15 @@ namespace Slicecam { } } + // a correction budget decides on the moves it commands, so it needs the + // confirmation that each one executed (the acamd move wait) + // + if ( this->fineacquire_state.max_adjustments > 0 && + !( this->fineacquire_state.move_timeout_sec > 0.0 ) ) { + logwrite( function, "ERROR FINE_ACQUIRE_MAX_ADJUSTMENTS > 0 needs FINE_ACQUIRE_MOVE_TIMEOUT > 0" ); + return ERROR; + } + // min_snr, when set, must be at least 2 // NAN (unset) disables SNR override // @@ -2101,6 +2318,10 @@ namespace Slicecam { // if ( whattodo == "stop" ) { this->should_framegrab_run.store( false, std::memory_order_release ); // tells framegrab loop to stop + { + std::lock_guard lock(this->acam_mtx); + this->acam_cv.notify_all(); + } if ( this->is_framegrab_running.load(std::memory_order_acquire) ) { // wait for it to stop int wait_ms = std::max( static_cast(3000*(this->camera.andor.begin()->second->camera_info.exptime+1)), 5000 ); // alert user that framegrabbing has stopped @@ -2180,6 +2401,10 @@ namespace Slicecam { // frame will be grabbed. // this->should_framegrab_run.store( false, std::memory_order_release ); + { + std::lock_guard lock(this->acam_mtx); + this->acam_cv.notify_all(); + } if ( this->is_framegrab_running.load(std::memory_order_acquire) ) return; } else @@ -2596,6 +2821,10 @@ namespace Slicecam { // bool is_guiding = this->is_acam_guiding.load(); + // arcsec for the log; the guiding path sends degrees + const double ra_off_arcsec = ra_off * 3600.; + const double dec_off_arcsec = dec_off * 3600.; + // send the offsets now // if ( is_guiding ) { @@ -2632,7 +2861,7 @@ namespace Slicecam { } std::ostringstream message; - message << "requested offsets dRA=" << ra_off << " dDEC=" << dec_off << " arcsec"; + message << "requested offsets dRA=" << ra_off_arcsec << " dDEC=" << dec_off_arcsec << " arcsec"; logwrite(function, message.str()); return NO_ERROR; diff --git a/slicecamd/slicecam_interface.h b/slicecamd/slicecam_interface.h index 90a7fdb1..922bdbcf 100644 --- a/slicecamd/slicecam_interface.h +++ b/slicecamd/slicecam_interface.h @@ -95,6 +95,9 @@ namespace Slicecam { int settle_frames = 0; ///< countdown of frames to discard while telescope settles int settle_count = 2; ///< configured: frames to discard after each move double settle_sec = 3.0; ///< time-based settle (sec) after each move; 0 disables (needed when autoexpose shortens frames so the frame-count settle is too brief in wall-clock) + int max_adjustments = 0; ///< correction budget: at most this many corrections per run; the last is sent only if the offset it removes is <= relaxed_factor x goal_arcsec, and the run converges once it has executed, else fails; 0 disables + double relaxed_factor = 2.0; ///< correction budget: multiple of goal_arcsec the last correction may remove + double move_timeout_sec = 60.0; ///< max wait (sec) for acamd to report the goal change applied; 0 disables the wait int consecutive_centroid_failures = 0; ///< counts consecutive centroid failures // exposure compensation (shared by the reactive trim and, later, autoexpose) double exptime_min = 0.1; ///< clamp: minimum auto-adjusted exposure (sec) @@ -149,6 +152,7 @@ namespace Slicecam { // the ACAM->slit pointing offset vs geometry over time. Touched only from the // single framegrab thread, so plain doubles are sufficient. double fineacq_total_dra = 0.0; ///< sum of applied dRA corrections this run [arcsec] + int fineacq_adjustments = 0; ///< corrections commanded this run (correction budget counter; survives FineAcqState::reset()) double fineacq_total_ddec = 0.0; ///< sum of applied dDEC corrections this run [arcsec] double fineacq_goal_ra = NAN; ///< per-run snapshot of goal RA [deg], taken at fineacquire start, logged at lock double fineacq_goal_dec = NAN; ///< per-run snapshot of goal DEC [deg], taken at fineacquire start, logged at lock @@ -193,6 +197,8 @@ namespace Slicecam { std::atomic is_acam_guiding; ///< is acam guiding? std::atomic last_acam_pubtime{0}; ///< pubtime (us) of latest received acamd status + std::atomic acam_goalshift_raised_us{0}; ///< pubtime (us) of the latest acamd status showing GOALSHIFT_PENDING set + std::atomic acam_goalshift_cleared_us{0}; ///< pubtime (us) of the latest acamd status showing GOALSHIFT_PENDING clear // Latest target (goal) coords published on Topic::TARGETINFO // NAN until a TARGETINFO arrives, so manual runs with no sequencer target log nan. diff --git a/tcsd/tcs_interface.cpp b/tcsd/tcs_interface.cpp index 1fe8bee7..c05d566c 100644 --- a/tcsd/tcs_interface.cpp +++ b/tcsd/tcs_interface.cpp @@ -1857,6 +1857,9 @@ namespace TCS { if ( cmd == "?NAME" || cmd == "LAMPS?" || cmd == "NPS" || + cmd == "?ONTARGET" || // reply IS the flag (0|1), not a status code: + // 0 collides with TCS_SUCCESS ("success") and + // 1 falls through to "tcs_undefined" ERROR cmd == "?PARALLACTIC" || cmd == "?WEATHER" || cmd == "RAWDEC" ||