68 image lysozyme dataset recorded on the Jungfrau 16M detector at SwissFEL and formatted as a NeXus file, revised for clean cnxvalidate error report

<p>This kit includes an additional revised master file, lyso009a_0087.JF07T32V01_master_rev.h5 that provides compliance with the October 2019 NXmx specification as proposed in https://github.com/HDRMX/definitions.git</p> <p>To create a new NeXus master file, assuming DIALS is installed in the folder $DIALS, use this command:</p> <p>libtbx.python $DIALS/modules/cctbx_project/xfel/swissfel/jf16m_cxigeom2nexus.py unassembled_file=lyso009a_0087.JF07T32V01.h5 geom_file=16M_bernina_backview_optimized_adu_quads.geom wavelength=1.368479 detector_distance=97.830 mask_file=lyso009a_0087.JF07T32V01.mask.h5</p> <p>Geometry file is in CrystFEL format but has been realigned to group the modules hierarchically into quadrants.</p> <p>View the data using DIALS: dials.image_viewer lyso009a_0087.JF07T32V01_master.h5</p> <p>Process the data using DIALS, treating the images as stills, assuming 64 cores available on the system:<br> dials.stills_process mp.nproc=64 lyso009a_0087.JF07T32V01_master.h5 dispersion.gain=10 known_symmetry.space_group=P43212 known_symmetry.unit_cell=77,77,37,90,90,90 refinement_protocol.d_min_start=2.5</p> <p>Download DIALS at&nbsp;dials.github.io.</p> <p>After the DIALS run, for full NXmx compliance you will need the jungfrau portions of the script that was used to generate lyso009a_0087.JF07T32V01_master_rev.h5</p> <p>&nbsp;</p> <pre>#!/bin/bash cp Therm_6_2.nxs Therm_6_2_rev.nxs cp Therm_6_2_master.h5 Therm_6_2_master_rev.h5 cp jungfrau/lyso009a_0087.JF07T32V01_master.h5 jungfrau/lyso009a_0087.JF07T32V01_master_rev.h5 export curdat=`date +%FT%T.%3` export LD_LIBRARY_PATH=$HOME/lib export HDF5_PLUGIN_PATH=$HOME/lib export PATH=$HOME/bin:$PATH h5copy -i Therm_6_2_rev.nxs -o Therm_6_2_master_rev.h5 -s /entry/instrument/name -d /entry/instrument/name -f ref h5copy -i Therm_6_2_rev.nxs -o Therm_6_2_master_rev.h5 -s /entry/instrument/source -d /entry/source -f ref h5copy -i Therm_6_2_rev.nxs -o Therm_6_2_rev.nxs -s /entry/instrument/source -d /entry/source -f ref h5copy -i jungfrau/lyso009a_0087.JF07T32V01_master.h5 -o jungfrau/lyso009a_0087.JF07T32V01_master_rev.h5 -s /entry/sample/beam -d /entry/instrument/beam -f ref export end_time=`h5dump -d &quot;/entry/end_time&quot; Therm_6_2_master.h5 | grep &quot;:&quot; | sed &#39;s/^.........//&#39;|sed &#39;s/.\$//&#39;` echo &quot;end_time: $end_time&quot; python &lt;&lt; &#39;EOL&#39; import h5py as h5 import numpy as np import os end_time=os.environ[&#39;end_time&#39;] curdat=os.environ[&#39;curdat&#39;] fvds = h5.File(&#39;Therm_6_2_rev.nxs&#39;,&#39;r+&#39;) fmaster = h5.File(&#39;Therm_6_2_master_rev.h5&#39;,&#39;r+&#39;) jungfrau= h5.File(&#39;jungfrau/lyso009a_0087.JF07T32V01_master_rev.h5&#39;,&#39;r+&#39;) fvds_keys=fvds.keys() fmaster_keys=fmaster.keys() jungfrau_keys=jungfrau.keys() fvds_entry=fvds[&#39;entry&#39;] fmaster_entry=fmaster[&#39;entry&#39;] jungfrau_entry=jungfrau[&#39;entry&#39;] fvds_entry_keys=fvds_entry.keys() fmaster_entry_keys=fmaster_entry.keys() jungfrau_entry_keys=jungfrau_entry.keys() fvds_entry_instrument=fvds[&#39;entry&#39;][&#39;instrument&#39;] fmaster_entry_instrument=fmaster[&#39;entry&#39;][&#39;instrument&#39;] jungfrau_entry_instrument=jungfrau[&#39;entry&#39;][&#39;instrument&#39;] fvds_entry_instrument_keys=fvds_entry_instrument.keys() fmaster_entry_instrument_keys=fmaster_entry_instrument.keys() jungfrau_entry_instrument_keys=jungfrau_entry_instrument.keys() fvds_entry_instrument_name=(fvds[&#39;entry&#39;][&#39;instrument&#39;][&#39;name&#39;]) fmaster_entry_instrument_name=(fmaster[&#39;entry&#39;][&#39;instrument&#39;][&#39;name&#39;]) jungfrau[&#39;entry&#39;][&#39;instrument&#39;].create_dataset(&quot;name&quot;, data=np.string_(&quot;Paul Scherrer Institute SwissFEL Aramis 1 (Alvra)&quot;)) jungfrau_entry_instrument_name=(jungfrau[&#39;entry&#39;][&#39;instrument&#39;][&#39;name&#39;]) fvds_entry_instrument_short_name=fvds_entry_instrument.attrs[&#39;short_name&#39;] fmaster_entry_instrument_short_name=fmaster_entry_instrument.attrs[&#39;short_name&#39;] jungfrau_entry_instrument_name.attrs.modify(&#39;short_name&#39;,np.string_(&quot;Alvra&quot;)) jungfrau_entry_instrument_short_name=jungfrau_entry_instrument_name.attrs[&#39;short_name&#39;] zero_offset=fmaster_entry_instrument[&#39;detector&#39;][&#39;module&#39;][&#39;fast_pixel_direction&#39;].attrs[&#39;offset&#39;] fmaster_det_z=fmaster_entry_instrument[&#39;transformations&#39;][&#39;det_z&#39;] fvds_det_z=fvds_entry_instrument[&#39;transformations&#39;][&#39;det_z&#39;] print(&#39;fvds_keys: &#39;,fvds_keys) print(&#39;fmaster_keys: &#39;,fmaster_keys) print(&#39;jungfrau_keys: &#39;,jungfrau_keys) print(&#39;fvds_entry_keys: &#39;,fvds_entry_keys) print(&#39;fmaster_entry_keys: &#39;,fmaster_entry_keys) print(&#39;jungfrau_entry_keys: &#39;,jungfrau_entry_keys) print(&#39;fvds_entry_instrument_keys: &#39;,fvds_entry_instrument_keys) print(&#39;fmaster_entry_instrument_keys: &#39;,fmaster_entry_instrument_keys) print(&#39;jungfrau_entry_instrument_keys: &#39;,jungfrau_entry_instrument_keys) print(&#39;fvds_entry_instrument_name: &#39;,fvds_entry_instrument_name) print(&#39;fmaster_entry_instrument_name: &#39;,fmaster_entry_instrument_name) print(&#39;jungfrau_entry_instrument_name: &#39;,jungfrau_entry_instrument_name) print(&#39;fvds_entry_instrument_short_name: &#39;,fvds_entry_instrument_short_name) print(&#39;fmaster_entry_instrument_short_name: &#39;,fmaster_entry_instrument_short_name) print(&#39;jungfrau_entry_instrument_short_name: &#39;,jungfrau_entry_instrument_short_name) print(&#39;fmaster_entry_instrument_detector_module_fast_pixel_direction_offset: &#39;,zero_offset) print(&#39;fmaster_entry_instrument_detector_detector_z_det_z: &#39;,fmaster_det_z) print(&#39;fmaster_entry_end_time: &#39;,end_time) fmaster.attrs.modify(&#39;file_time&#39;,np.string_(end_time)) fmaster.attrs.modify(&#39;file_name&#39;,np.string_(&#39;Therm_6_2_master_rev.h5&#39;)) fmaster.attrs.modify(&#39;HDF5_Version&#39;,np.string_(&#39;hdf5-1.8.18&#39;)) fvds.attrs.modify(&#39;file_time&#39;,np.string_(end_time)) fvds.attrs.modify(&#39;file_name&#39;,np.string_(&#39;Therm_6_2_master_rev.h5&#39;)) fvds.attrs.modify(&#39;HDF5_Version&#39;,np.string_(&#39;hdf5-1.10.5&#39;)) jungfrau.attrs.modify(&#39;file_time&#39;,np.string_(curdat)) jungfrau.attrs.modify(&#39;file_name&#39;,np.string_(&#39;lyso009a_0087.JF07T32V01_master.h5&#39;)) jungfrau.attrs.modify(&#39;HDF5_Version&#39;,np.string_(&#39;hdf5-1.10.5&#39;)) fvds_entry_instrument_name.attrs.modify(&#39;short_name&#39;,np.string_(fvds_entry_instrument.attrs[&#39;short_name&#39;])) fmaster_entry_instrument_name.attrs.modify(&#39;short_name&#39;,np.string_(fmaster_entry_instrument.attrs[&#39;short_name&#39;])) fmaster_entry_instrument[&#39;attenuator&#39;][&#39;attenuator_transmission&#39;].attrs.modify(&#39;units&#39;,np.string_(&quot;&quot;)) fmaster_entry_instrument[&#39;detector&#39;][&#39;count_time&#39;].attrs.modify(&#39;units&#39;,np.string_(&quot;s&quot;)) fvds_entry_instrument_name.attrs.modify(&#39;short_name&#39;,np.string_(fvds_entry_instrument.attrs[&#39;short_name&#39;])) fvds_entry_instrument[&#39;attenuator&#39;][&#39;attenuator_transmission&#39;].attrs.modify(&#39;units&#39;,np.string_(&quot;&quot;)) fvds_entry_instrument[&#39;detector&#39;][&#39;count_time&#39;].attrs.modify(&#39;units&#39;,np.string_(&quot;s&quot;)) fmaster_det_z.attrs.modify(&#39;offset&#39;,zero_offset) fvds_det_z.attrs.modify(&#39;offset&#39;,zero_offset) fmaster_entry[&#39;sample&#39;][&#39;transformations&#39;][&#39;phi&#39;].attrs.modify(&#39;offset&#39;,zero_offset) fmaster_entry[&#39;sample&#39;][&#39;transformations&#39;][&#39;chi&#39;].attrs.modify(&#39;offset&#39;,zero_offset) fmaster_entry[&#39;sample&#39;][&#39;transformations&#39;][&#39;sam_x&#39;].attrs.modify(&#39;offset&#39;,zero_offset) fmaster_entry[&#39;sample&#39;][&#39;transformations&#39;][&#39;sam_y&#39;].attrs.modify(&#39;offset&#39;,zero_offset) fmaster_entry[&#39;sample&#39;][&#39;transformations&#39;][&#39;sam_z&#39;].attrs.modify(&#39;offset&#39;,zero_offset) fmaster_entry[&#39;sample&#39;][&#39;transformations&#39;][&#39;omega&#39;].attrs.modify(&#39;offset&#39;,zero_offset) fvds_entry[&#39;sample&#39;][&#39;transformations&#39;][&#39;phi&#39;].attrs.modify(&#39;offset&#39;,zero_offset) fvds_entry[&#39;sample&#39;][&#39;transformations&#39;][&#39;chi&#39;].attrs.modify(&#39;offset&#39;,zero_offset) fvds_entry[&#39;sample&#39;][&#39;transformations&#39;][&#39;sam_x&#39;].attrs.modify(&#39;offset&#39;,zero_offset) fvds_entry[&#39;sample&#39;][&#39;transformations&#39;][&#39;sam_y&#39;].attrs.modify(&#39;offset&#39;,zero_offset) fvds_entry[&#39;sample&#39;][&#39;transformations&#39;][&#39;sam_z&#39;].attrs.modify(&#39;offset&#39;,zero_offset) fvds_entry[&#39;sample&#39;][&#39;transformations&#39;][&#39;omega&#39;].attrs.modify(&#39;offset&#39;,zero_offset) print(fmaster[&#39;entry&#39;][&#39;instrument&#39;][&#39;name&#39;].attrs[&#39;short_name&#39;]) print(fmaster[&#39;entry&#39;][&#39;instrument&#39;][&#39;name&#39;].attrs[&#39;short_name&#39;].shape) print(fmaster[&#39;entry&#39;][&#39;instrument&#39;][&#39;name&#39;].attrs[&#39;short_name&#39;].dtype) print(&quot;/entry/instrument/ELE_D0/pixel_mask_applied :&quot;,jungfrau_entry_instrument[&#39;ELE_D0&#39;][&#39;pixel_mask_applied&#39;]) del jungfrau_entry_instrument[&#39;ELE_D0&#39;][&#39;pixel_mask_applied&#39;] jungfrau_entry_instrument[&#39;ELE_D0&#39;].create_dataset(&quot;pixel_mask_applied&quot;,dtype=&#39;int8&#39;, data=1) print(&quot;/entry/instrument/ELE_D0/pixel_mask_applied :&quot;,jungfrau_entry_instrument[&#39;ELE_D0&#39;][&#39;pixel_mask_applied&#39;]) jungfrau_entry_source=jungfrau_entry.create_group(&#39;source&#39;) jungfrau_entry_source=jungfrau_entry[&#39;source&#39;] jungfrau_entry_source.attrs.modify(&#39;NX_class&#39;,np.string_(&quot;NXsource&quot;)) jungfrau_entry_source.create_dataset(&quot;name&quot;,data=np.string_(&quot;Paul Scherrer Institute SwissFEL&quot;)) jungfrau_entry_source[&#39;name&#39;].attrs.modify(&#39;short_name&#39;,np.string_(&quot;SwissFEL&quot;)) #jungfrau_entry_instrument.create_group[&#39;beam&#39;] #jungfrau_entry_instrument[&#39;beam&#39;]=jungfrau_entry[&#39;sample&#39;][&#39;beam&#39;] jungfrau_entry_instrument[&#39;beam&#39;].create_dataset(&#39;total_flux&#39;,dtype=&#39;float64&#39;,data=1000000000000.) jungfrau_entry_instrument[&#39;beam&#39;][&#39;total_flux&#39;].attrs.modify(&#39;units&#39;,np.string_(&#39;/pulse&#39;)) del jungfrau_entry[&#39;sample&#39;][&#39;beam&#39;] del fvds_entry_instrument.attrs[&#39;short_name&#39;] del fmaster_entry_instrument.attrs[&#39;short_name&#39;] del fmaster_entry_instrument[&#39;source&#39;] fvds.close() fmaster.close() jungfrau.close() quit() EOL $HOME/bin/nxvalidate -a NXmx -l /home/yaya/hdrmx_rev_29Sep19/hdrmx/definitions Therm_6_2_master_rev.h5 $HOME/bin/nxvalidate -a NXmx -l /home/yaya/hdrmx_rev_29Sep19/hdrmx/definitions Therm_6_2_rev.nxs $HOME/bin/nxvalidate -a NXmx -l /home/yaya/hdrmx_rev_29Sep19/hdrmx/definitions jungfrau/lyso009a_0087.JF07T32V01_master_rev.h5 </pre> <p>&nbsp;</p>