lsst.pipe.drivers  8.5-hsc+2
sourceTableDriver.py
Go to the documentation of this file.
1 from lsst.pipe.base import ArgumentParser
2 from lsst.pipe.tasks.postprocess import WriteSourceTableTask, TransformSourceTableTask
3 from lsst.pex.config import Config, Field, ConfigurableField, ListField
4 from lsst.ctrl.pool.parallel import BatchParallelTask, BatchTaskRunner
5 
6 
7 class SourceTableDriverConfig(Config):
8  writeSourceTable = ConfigurableField(
9  target=WriteSourceTableTask,
10  doc="Task to make parquet table for full src catalog",
11  )
12  transformSourceTable = ConfigurableField(
13  target=TransformSourceTableTask,
14  doc="Transform Source Table to DPDD specification",
15  )
16  ignoreCcdList = ListField(
17  dtype=int,
18  default=[],
19  doc="List of CCDs to ignore when processing",
20  )
21  ccdKey = Field(
22  dtype=str,
23  default="ccd",
24  doc="DataId key corresponding to a single sensor",
25  )
26 
27  def setDefaults(self):
28  self.writeSourceTable.doApplyExternalPhotoCalib=True
29  self.writeSourceTable.doApplyExternalSkyWcs=True
30 
31 
33  """Convert existing src tables to parquet Source Table
34 
35  This driver can convert PDR2-era `src` tables that do not have
36  * local photo calib columns
37  * local wcs columns
38  * sky_source flag and
39  * detect_isPrimary flags set.set
40 
41  It is specialized for the 2021 HSC data release in which we will
42  not rerun singleFrameDriver but start the processing from FGCM.
43 
44  Can be removed during after the Gen2 deprecation period.
45  """
46  ConfigClass = SourceTableDriverConfig
47  _DefaultName = "sourceTableDriver"
48  RunnerClass = BatchTaskRunner
49 
50  def __init__(self, *args, **kwargs):
51  """!
52  Constructor
53 
54  @param[in,out] kwargs other keyword arguments for lsst.ctrl.pool.BatchParallelTask
55  """
56  BatchParallelTask.__init__(self, *args, **kwargs)
57  self.ignoreCcds = set(self.config.ignoreCcdList)
58  self.makeSubtask("writeSourceTable")
59  self.makeSubtask("transformSourceTable")
60 
61  @classmethod
62  def _makeArgumentParser(cls, *args, **kwargs):
63  kwargs.pop("doBatch", False)
64  parser = ArgumentParser(name="sourceTableDriver", *args, **kwargs)
65  parser.add_id_argument("--id",
66  datasetType="src",
67  level="sensor",
68  help="data ID, e.g. --id visit=12345 ccd=67")
69  return parser
70 
71  def runDataRef(self, sensorRef):
72  """Process a single CCD
73  """
74  if sensorRef.dataId[self.config.ccdKey] in self.ignoreCcds:
75  self.log.warn("Ignoring %s: CCD in ignoreCcdList" %
76  (sensorRef.dataId))
77  return None
78 
79  with self.logOperation("processing %s" % (sensorRef.dataId,)):
80  res = self.writeSourceTable.runDataRef(sensorRef)
81  df = self.transformSourceTable.run(res.table,
82  funcs=self.transformSourceTable.getFunctors(),
83  dataId=sensorRef.dataId)
84  self.transformSourceTable.write(df, sensorRef)
85  return df
86 
87  def _getMetadataName(self):
88  """There's no metadata to write out"""
89  return None
def logOperation(self, operation, catch=False, trace=True)