diff --git a/requirements.txt b/requirements.txt index 61bbe80b..83698a25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,11 +5,8 @@ scipy matplotlib lmfit pandas -# Can install sotodlib from PyPI once Python version in pysmurf-controller is >=3.10 -# i.e. when [1] is merged and propagated to [2]. -# [1] - https://github.com/simonsobs/smurf_dockers/pull/6 -# [2] - https://github.com/simonsobs/socs/blob/main/docker/pysmurf_controller/Dockerfile -sotodlib @ git+https://github.com/simonsobs/sotodlib.git@5d613d5915b1716c401abecb5446088bce5fc1a4 +zmq +sotodlib pysmurf-slac # scripts diff --git a/sodetlib/__init__.py b/sodetlib/__init__.py index 359f45cb..931b5a50 100644 --- a/sodetlib/__init__.py +++ b/sodetlib/__init__.py @@ -1,7 +1,6 @@ import os from functools import wraps try: - import epics from pysmurf.client.util.pub import set_action except Exception as e: # Just return base function regularly if can't import set_action diff --git a/sodetlib/det_config.py b/sodetlib/det_config.py index 58c22170..907ddcba 100644 --- a/sodetlib/det_config.py +++ b/sodetlib/det_config.py @@ -117,11 +117,8 @@ def odict_rep(dumper, data): # Gradient Descent Parameters "gradientDescentMaxIters": 100, "gradientDescentAverages": 2, - "gradientDescentGain": 0.001, "gradientDescentConvergeHz": 500, "gradientDescentStepHz": 5000, - "gradientDescentMomentum": 1, - "gradientDescentBeta": 0.1, # Fixed tones "fixed_tones": {"enabled": False, "freq_offsets": [], "channels": [], "tone_power": 0}, @@ -640,22 +637,22 @@ def dump_configs(self, output_dir=None, clobber=False, dump_rogue_tree=False): return outfiles - def get_smurf_control(self, offline=False, epics_root=None, + def get_smurf_control(self, offline=False, server_port=None, smurfpub_id=None, make_logfile=False, setup=False, dump_configs=None, config_dir=None, apply_dev_configs=False, load_device_tune=True, **pysmurf_kwargs): """ Creates pysmurf instance based off of configuration parameters. - If not specified as keyword arguments ``epics_root`` and ``smurf_pub`` + If not specified as keyword arguments ``smurf_pub`` will be created based on the slot and crate id's. Args: offline (bool): Whether to start pysmurf in offline mode. Defaults to False - epics_root (str, optional): - Pysmurf epics root. If none, it will be set to - ``smurf_server_s``. + server_port (int, optional): + The port for the SMuRF server to connect to. Will infer from + slot number if not provided. smurfpub_id (str, optional): Pysmurf publisher ID. If None, will default to crate_slot. @@ -674,12 +671,12 @@ def get_smurf_control(self, offline=False, epics_root=None, import pysmurf.client slot_cfg = self.sys['slots'][f'SLOT[{self.slot}]'] - if epics_root is None: - epics_root = f'smurf_server_s{self.slot}' if smurfpub_id is None: smurfpub_id = self.stream_id if dump_configs is None: dump_configs = self.dump + if server_port is None: + server_port = 9000 + 3 * int(self.slot) # Pysmurf publisher will check this to determine publisher id. os.environ['SMURFPUB_ID'] = smurfpub_id @@ -688,9 +685,10 @@ def get_smurf_control(self, offline=False, epics_root=None, S = pysmurf.client.SmurfControl(offline=True) else: S = pysmurf.client.SmurfControl( - epics_root=epics_root, cfg_file=self.pysmurf_file, setup=setup, + cfg_file=self.pysmurf_file, setup=setup, make_logfile=make_logfile, data_path_id=smurfpub_id, - **pysmurf_kwargs) + server_port=server_port, **pysmurf_kwargs + ) self.S = S # Lets just stash this in pysmurf... S._sodetlib_cfg = self diff --git a/sodetlib/hammers/jackhammer.py b/sodetlib/hammers/jackhammer.py index e36a74da..f0bdeb60 100644 --- a/sodetlib/hammers/jackhammer.py +++ b/sodetlib/hammers/jackhammer.py @@ -7,6 +7,7 @@ import os import threading from typing import List, Literal +import zmq class TermColors: @@ -108,36 +109,47 @@ def util_run(cmd, args=[], name=None, rm=True, **run_kwargs): return subprocess.run(shlex.split(cmd), cwd=cwd, **run_kwargs) -def check_epics_connection(epics_server, retry=False): +def check_server_connection(server_port, retry=False, timeout=1): """ - Checks if we can connect to a specific epics server. + Checks if we can connect to a specific server. Args: - epics_server (string): - epics server to connect to + server_port (int): + ZMQ server to connect to retry (bool): If true, will continuously check until a connection has been established. """ + # setup connection + c = zmq.Context() + + # message to check if server is ready + msg = {'path': 'AMCc.Ready', 'attr': 'get', 'args': [], 'kwargs': {}} + def do_ping(): + s = c.socket(zmq.REQ) + s.setsockopt(zmq.RCVTIMEO, timeout * 1000) + s.setsockopt(zmq.LINGER, 0) # discard undelivered messages + s.connect(f"tcp://localhost:{server_port + 1}") + try: + s.send_pyobj(msg) + resp = s.recv_pyobj() + return bool(resp) + except zmq.error.Again: + return False + finally: + s.close() + if retry: - print(f"Waiting for epics connection to {epics_server}", end='', flush=True) + print(f"Waiting for connection to server on port {server_port}", end='', flush=True) while True: - x = util_run( - 'caget', args=[f'{epics_server}:AMCc:enable'], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - if "True" in x.stdout.decode(): + if do_ping(): break print('.', end='', flush=True) print("\nConnected!") return True else: - x = util_run( - 'caget', args=[f'{epics_server}:AMCc:enable'], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - return "True" in x.stdout.decode() + return do_ping() def get_running_dockers(get_all=True): @@ -243,13 +255,13 @@ def dump_docker_logs(slots, dump_rogue_tree=False): if dump_rogue_tree: dump_script = '/sodetlib/scripts/dump_rogue_state.py' for slot in slots: - if check_epics_connection(f'smurf_server_s{slot}', retry=False): + if check_server_connection(9000 + 3 * slot, retry=False): out_file = os.path.join(dump_dir, f'rogue_state_s{slot}.yml') cprint(f"Dumping s{slot} state to {out_file}", style=TermColors.HEADER) util_run('python3', args=[dump_script, str(slot), out_file], name=f'rogue_dump_s{slot}') else: - print(f"Could not connect to epics for slot {slot}") + print(f"Could not connect to server for slot {slot}") def run_on_shelf_manager(cmd_str): """ Runs a command on the shelf manager. Takes in the command as a string""" @@ -474,8 +486,7 @@ def hammer_func(args): # Waits for streamer-dockers to start print("Waiting for server dockers to connect. This might take a few minutes...") for slot in slots: - epics_server = f'smurf_server_s{slot}' - check_epics_connection(epics_server, retry=True) + check_server_connection(9000 + 3 * slot, retry=True) if reboot and not args.skip_setup: cprint("Configuring pysmurf", style=TermColors.HEADER) @@ -564,7 +575,7 @@ def gui_func(args): slot = args.slot else: slot = available_slots[0] - server_port = 9000 + 2*slot + server_port = 9000 + 3*slot sodetlib_root = os.environ.get('SODETLIB_ROOT', '/home/cryo/sodetlib') script_path = os.path.join(sodetlib_root, 'hammers', 'run_gui.sh') diff --git a/sodetlib/operations/uxm_relock.py b/sodetlib/operations/uxm_relock.py index 5d13161f..e3f3036f 100644 --- a/sodetlib/operations/uxm_relock.py +++ b/sodetlib/operations/uxm_relock.py @@ -65,8 +65,7 @@ def reload_tune(S, cfg, bands, setup_notches=False, @sdl.set_action() def run_grad_descent_and_eta_scan( - S, cfg, bands=None, update_tune=False, force_run=False, max_iters=None, - gain=None): + S, cfg, bands=None, update_tune=False, force_run=False, max_iters=None): """ This function runs serial gradient and eta scan for each band. Critically, it pulls in gradient descent tune parameters from the device @@ -108,14 +107,10 @@ def run_grad_descent_and_eta_scan( if max_iters is None: max_iters = bcfg['gradientDescentMaxIters'] - if gain is None: - gain = bcfg['gradientDescentGain'] S.set_gradient_descent_step_hz(b, bcfg['gradientDescentStepHz']) S.set_gradient_descent_max_iters(b, max_iters) - S.set_gradient_descent_gain(b, gain) S.set_gradient_descent_converge_hz(b, bcfg['gradientDescentConvergeHz']) - S.set_gradient_descent_beta(b, bcfg['gradientDescentBeta']) S.log(f"Running grad descent and eta scan on band {b}") diff --git a/sodetlib/operations/uxm_setup.py b/sodetlib/operations/uxm_setup.py index 9f0530ab..2b60e841 100644 --- a/sodetlib/operations/uxm_setup.py +++ b/sodetlib/operations/uxm_setup.py @@ -145,7 +145,7 @@ def find_drain_voltage(S, target_Id, amp_name, vd_min=0.1, vd_max=0.95, @sdl.set_action() -def setup_amps(S, cfg, update_cfg=True, enable_300K_LNA=True): +def setup_amps(S, cfg, update_cfg=True, enable_300K_LNA=True, opt_args=None): """ Initial setup for 50k and hemt amplifiers. For C04/C05 cryocards, will first check if the drain voltages are set. Then checks if drain @@ -175,11 +175,16 @@ def setup_amps(S, cfg, update_cfg=True, enable_300K_LNA=True): If true, will update the device cfg and save the file. enable_300K_LNA: If true, will turn on the 300K LNAs. + opt_args : dict (optional) + Extra kwargs to pass to the `find_gate_voltage` calls. """ sdl.pub_ocs_log(S, "Starting setup_amps") exp = cfg.dev.exp + if opt_args is None: + opt_args = {} + # Determine cryocard rev major, minor, patch = S.C.get_fw_version() if major == 4: @@ -223,8 +228,13 @@ def setup_amps(S, cfg, update_cfg=True, enable_300K_LNA=True): if Vd != exp[f"amp_{amp}_drain_volt"]: S.set_amp_drain_voltage(amp, exp[f"amp_{amp}_drain_volt"]) + # extra args for optimisation + if "wait_time" not in opt_args: + opt_args["wait_time"] = exp['amp_step_wait_time'] # Check drain currents / scan bias voltages for amp in amp_list: + # arguments to pass to optimization function + amp_opts = opt_args.copy() delta_Id = np.abs(amp_biases[f"{amp}_drain_current"] - exp[f"amp_{amp}_drain_current"]) if delta_Id > exp[f'amp_{amp}_drain_current_tolerance']: # Only scan Vd if connected to an ASU hemt @@ -235,20 +245,23 @@ def setup_amps(S, cfg, update_cfg=True, enable_300K_LNA=True): ): S.log(f"{amp} current not within tolerance, scanning for correct drain voltage") success = find_drain_voltage( - S, exp[f"amp_{amp}_drain_current"], amp, wait_time=exp['amp_step_wait_time'], - id_tolerance=exp[f'amp_{amp}_drain_current_tolerance'], + S, exp[f"amp_{amp}_drain_current"], amp, + id_tolerance=exp[f'amp_{amp}_drain_current_tolerance'], **amp_opts ) else: S.log(f"{amp} current not within tolerance, scanning for correct gate voltage") + if "vg_min" not in amp_opts: + amp_opts["vg_min"] = exp[f'amp_{amp}_gate_volt_min'] + if "vg_max" not in opt_args: + amp_opts["vg_max"] = exp[f'amp_{amp}_gate_volt_max'], # If optimal value was found previously, use it first. init_gate_volt = exp[f'amp_{amp}_gate_volt'] if init_gate_volt is None: init_gate_volt = exp[f'amp_{amp}_init_gate_volt'] S.set_amp_gate_voltage(amp, init_gate_volt,override=True) success = find_gate_voltage( - S, exp[f"amp_{amp}_drain_current"], amp, wait_time=exp['amp_step_wait_time'], - id_tolerance=exp[f'amp_{amp}_drain_current_tolerance'], - vg_min=exp[f'amp_{amp}_gate_volt_min'], vg_max=exp[f'amp_{amp}_gate_volt_max'], + S, exp[f"amp_{amp}_drain_current"], amp, + id_tolerance=exp[f'amp_{amp}_drain_current_tolerance'], **amp_opts ) if not success: sdl.pub_ocs_log(S, f"Failed determining {amp} bias voltage") diff --git a/sodetlib/stream.py b/sodetlib/stream.py index a2d04123..30f4e6b6 100644 --- a/sodetlib/stream.py +++ b/sodetlib/stream.py @@ -231,8 +231,8 @@ def stream_g3_on(S, make_freq_mask=False, emulator=False, tag=None, reg.open_g3stream.set(1) # Sometimes it takes a bit for data to propogate through to the - # streamer - for _ in range(10): + # streamer. Wait up to 30s + for _ in range(60): sess_id = reg.g3_session_id.get() if sess_id != 0: break diff --git a/sodetlib/util.py b/sodetlib/util.py index f960f7c9..c797e7e0 100644 --- a/sodetlib/util.py +++ b/sodetlib/util.py @@ -15,7 +15,6 @@ if not os.environ.get('NO_PYSMURF', False): try: - import epics import pysmurf from pysmurf.client.command.cryo_card import cmd_make except Exception: @@ -208,8 +207,8 @@ def get_metadata(S, cfg): 'iv_file': cfg.dev.exp.get('iv_file'), 'v_bias': S.get_tes_bias_bipolar_array(), 'pysmurf_client_version': pysmurf.__version__, - 'rogue_version': S._caget(f'{S.epics_root}:AMCc:RogueVersion'), - 'smurf_core_version': S._caget(f'{S.epics_root}:AMCc:SmurfApplication:SmurfVersion'), + 'rogue_version': S._caget('AMCc.RogueVersion'), + 'smurf_core_version': S._caget('AMCc.SmurfApplication.SmurfVersion'), 'sodetlib_version': sodetlib.__version__, 'fpga_git_hash': S.get_fpga_git_hash_short(), 'cryocard_fw_version': S.C.get_fw_version(), @@ -541,7 +540,7 @@ def get_r2(sig, sig_hat): class _Register: def __init__(self, S, addr): self.S = S - self.addr = S.epics_root + ":" + addr + self.addr = addr def get(self, **kw): return self.S._caget(self.addr, **kw) @@ -555,11 +554,11 @@ class Registers: they are not in the standard rogue tree, or settable by existing pysmurf get/set functions """ - _root = 'AMCc:' - _processor = _root + "SmurfProcessor:" - _sostream = _processor + "SOStream:" - _sofilewriter = _sostream + 'SOFileWriter:' - _source_root = _root + 'StreamDataSource:' + _root = 'AMCc.' + _processor = _root + "SmurfProcessor." + _sostream = _processor + "SOStream." + _sofilewriter = _sostream + 'SOFileWriter.' + _source_root = _root + 'StreamDataSource.' _registers = { 'pysmurf_action': _sostream + 'pysmurf_action', @@ -673,25 +672,19 @@ def set_current_mode(S, bgs, mode, const_current=True): nbits = S._rtm_slow_dac_nbits dac_data = np.clip(dac_data, -2**(nbits-1), 2**(nbits-1)-1) - dac_data_reg = S.rtm_spi_max_root + S._rtm_slow_dac_data_array_reg - - - if isinstance(S.C.writepv, str): - cryocard_writepv = S.C.writepv - else: - cryocard_writepv = S.C.writepv.pvname + dac_data_reg = S.rtm_spi_max_root + S._rtm_slow_dac_data_reg # It takes longer for DC voltages to settle than it does to toggle the # high-current relay, so we can set them at the same time when switchign # to hcm, but when switching to lcm we need a sleep statement to prevent # dets from latching. if mode: - epics.caput_many([cryocard_writepv, dac_data_reg], [relay_data, dac_data], - wait=True) + S._caput(dac_data_reg, dac_data, wait_done=False) + S.C.do_write(S.C.relay_address, new_relay) else: S._caput(dac_data_reg, dac_data) time.sleep(0.04) - S._caput(cryocard_writepv, relay_data) + S.C.do_write(S.C.relay_address, new_relay) time.sleep(0.1) # Just to be safe