Skip to content

Commit

Permalink
Merge pull request #251 from jadball/scanbooks
Browse files Browse the repository at this point in the history 
Removing functions from processing notebooks
  • Loading branch information
@jonwright
jonwright authored Mar 13, 2024
2 parents fe74de9 + 9a532e5 commit 8ae3999
Show file tree
Hide file tree
Showing 19 changed files with 1,750 additions and 2,092 deletions.
86 changes: 66 additions & 20 deletions ImageD11/nbGui/3DXRD/0_3DXRD_segment_frelon.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,13 +152,27 @@
},
"outputs": [],
"source": [
"# NOTE: For old datasets before the new directory layout structure, we don't distinguish between RAW_DATA and PROCESSED_DATA\n",
"\n",
"### USER: specify your experimental directory\n",
"\n",
"rawdata_path = \"/home/esrf/james1997a/Data/ihma439/id11/20231211/RAW_DATA\"\n",
"rawdata_path = \"/data/visitor/ihma439/id11/20231211/RAW_DATA\"\n",
"\n",
"!ls -lrt {rawdata_path}\n",
"\n",
"processed_data_root_dir = \"/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/James/20240226\" # USER: modify this to change the destination folder if desired"
"### USER: specify where you want your processed data to go\n",
"\n",
"processed_data_root_dir = \"/data/visitor/ihma439/id11/20231211/PROCESSED_DATA/James/nb_testing\"# NOTE: For old datasets before the new directory layout structure, we don't distinguish between RAW_DATA and PROCESSED_DATA\n",
"\n",
"### USER: specify your experimental directory\n",
"\n",
"rawdata_path = \"/data/visitor/ihma439/id11/20231211/RAW_DATA\"\n",
"\n",
"!ls -lrt {rawdata_path}\n",
"\n",
"### USER: specify where you want your processed data to go\n",
"\n",
"processed_data_root_dir = \"/data/visitor/ihma439/id11/20231211/PROCESSED_DATA/James/nb_testing\""
]
},
{
Expand Down Expand Up @@ -211,9 +225,24 @@
},
"outputs": [],
"source": [
"# USER: specify path to background file\n",
"# USER: specify path to background and mask file\n",
"\n",
"bg_file = \"/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/FeAu_0p5_tR/tdxrd_all/ff_bkg.edf\""
"bg_file = \"/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/FeAu_0p5_tR/tdxrd_all/ff_bkg.edf\"\n",
"maskfile = '/data/id11/inhouse1/ewoks/detectors/files/Frelon2k_C36/mask.edf'"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e853b7db-1d0b-4504-a6a6-e6b38c5ff32d",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"ds.splinefile = spline_file\n",
"ds.maskfile = maskfile\n",
"ds.bgfile = bg_file"
]
},
{
Expand All @@ -227,7 +256,8 @@
"source": [
"#Define the initial parameters\n",
"start_worker_args = {\n",
" \"bgfile\":bg_file,\n",
" \"bgfile\":ds.bgfile,\n",
" \"maskfile\":ds.maskfile,\n",
" \"threshold\":50,\n",
" \"smoothsigma\":1.0,\n",
" \"bgc\":0.9,\n",
Expand Down Expand Up @@ -274,7 +304,8 @@
"\n",
"\n",
"def update(threshold, smoothsigma, bgc, minpx, m_offset_thresh, m_ratio_thresh):\n",
" image_worker = worker(bg_file,\n",
" image_worker = worker(ds.bgfile,\n",
" ds.maskfile,\n",
" threshold,\n",
" smoothsigma,\n",
" bgc,\n",
Expand Down Expand Up @@ -310,7 +341,8 @@
"outputs": [],
"source": [
"end_worker_args = {\n",
" \"bgfile\":bg_file,\n",
" \"bgfile\":ds.bgfile,\n",
" \"maskfile\":ds.maskfile,\n",
" \"threshold\":thresh_slider.value,\n",
" \"smoothsigma\":smsig_slider.value,\n",
" \"bgc\":bgc_slider.value,\n",
Expand Down Expand Up @@ -345,7 +377,7 @@
"\n",
"nthreads = len(os.sched_getaffinity(os.getpid()))\n",
"\n",
"cf_2d, cf_3d = process(ds, bg_file, nthreads-1, end_worker_args)"
"cf_2d, cf_3d = process(ds, nthreads-1, end_worker_args)"
]
},
{
Expand Down Expand Up @@ -421,7 +453,7 @@
},
"outputs": [],
"source": [
"parfile = '/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/James/20240226/Fe_tdxrd_refined.par'"
"parfile = '/home/esrf/james1997a/Data/ihma439/id11/20231211/SCRIPTS/James/3DXRD/Fe_tdxrd_refined.par'"
]
},
{
Expand Down Expand Up @@ -453,6 +485,19 @@
"ImageD11.columnfile.colfile_to_hdf(cf_3d, ds.col3dfile)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7085183c-1991-49b3-af09-abe119542166",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"ds.parfile = parfile\n",
"ds.save()"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand All @@ -472,7 +517,7 @@
"metadata": {},
"outputs": [],
"source": [
"# Now that we're happy with our indexing parameters, we can run the below cell to do this in bulk for many samples/datasets\n",
"# Now that weparfile happy with our indexing parameters, we can run the below cell to do this in bulk for many samples/datasets\n",
"# by default this will do all samples in sample_list, all datasets with a prefix of dset_prefix\n",
"# you can add samples and datasets to skip in skips_dict\n",
"\n",
Expand All @@ -489,12 +534,6 @@
"# manual override:\n",
"# samples_dict = {\"FeAu_0p5_tR_nscope\": [\"top_100um\", \"top_200um\"]}\n",
"\n",
"mask_path = '/data/id11/inhouse1/ewoks/detectors/files/Frelon2k_C36/mask.edf'\n",
"\n",
"spline_file = '/data/id11/inhouse1/ewoks/detectors/files/Frelon2k_C36/frelon36.spline'\n",
"parfile = '/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/FeAu_0p5_tR/tdxrd_all/fitted.par'\n",
"bg_file = \"/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/FeAu_0p5_tR/tdxrd_all/ff_bkg.edf\"\n",
"\n",
"worker_args = end_worker_args\n",
"\n",
"nthreads = len(os.sched_getaffinity(os.getpid()))\n",
Expand All @@ -518,13 +557,17 @@
" ds.import_all(scans=[\"1.1\"])\n",
" print(f\"I have a DataSet {ds.dset} in sample {ds.sample}\")\n",
" ds.save()\n",
" \n",
" ds.splinefile = spline_file\n",
" ds.maskfile = maskfile\n",
" ds.bgfile = bg_file\n",
"\n",
" print(\"Peaksearching\")\n",
" cf_2d, cf_3d = process(ds, bg_file, nthreads-1, worker_args)\n",
" cf_2d, cf_3d = process(ds, nthreads-1, worker_args)\n",
" \n",
" print(\"Spatially correcting peaks\")\n",
" cf_2d = utils.apply_spatial(cf_2d, spline_file, nthreads)\n",
" cf_3d = utils.apply_spatial(cf_3d, spline_file, nthreads)\n",
" cf_2d = utils.apply_spatial_lut(cf_2d, spline_file)\n",
" cf_3d = utils.apply_spatial_lut(cf_3d, spline_file)\n",
" \n",
" print(\"Saving peaks to file\")\n",
" cf_2d.parameters.loadparameters(parfile)\n",
Expand All @@ -534,7 +577,10 @@
" \n",
" cf_3d.parameters.loadparameters(parfile)\n",
" cf_3d.updateGeometry()\n",
" ImageD11.columnfile.colfile_to_hdf(cf_3d, ds.col3dfile)"
" ImageD11.columnfile.colfile_to_hdf(cf_3d, ds.col3dfile)\n",
" \n",
" ds.parfile = parfile\n",
" ds.save()"
]
},
{
Expand Down
50 changes: 25 additions & 25 deletions ImageD11/nbGui/3DXRD/1_3DXRD_refine_parameters.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,13 +86,17 @@
},
"outputs": [],
"source": [
"# NOTE: For old datasets before the new directory layout structure, we don't distinguish between RAW_DATA and PROCESSED_DATA\n",
"\n",
"### USER: specify your experimental directory\n",
"\n",
"rawdata_path = \"/home/esrf/james1997a/Data/ihma439/id11/20231211/RAW_DATA\"\n",
"rawdata_path = \"/data/visitor/ihma439/id11/20231211/RAW_DATA\"\n",
"\n",
"!ls -lrt {rawdata_path}\n",
"\n",
"processed_data_root_dir = \"/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/James/20240226\" # USER: modify this to change the destination folder if desired"
"### USER: specify where you want your processed data to go\n",
"\n",
"processed_data_root_dir = \"/data/visitor/ihma439/id11/20231211/PROCESSED_DATA/James/nb_testing\""
]
},
{
Expand All @@ -106,11 +110,7 @@
"# USER: pick a sample and a dataset you want to segment\n",
"\n",
"sample = \"FeAu_0p5_tR\"\n",
"dataset = \"ff1\"\n",
"\n",
"# USER: specify path to detector mask\n",
"\n",
"mask_path = '/data/id11/inhouse1/ewoks/detectors/files/Frelon2k_C36/mask.edf'"
"dataset = \"ff1\""
]
},
{
Expand All @@ -123,12 +123,7 @@
"source": [
"# desination of H5 files\n",
"\n",
"dset_path = os.path.join(processed_data_root_dir, sample, f\"{sample}_{dataset}\", f\"{sample}_{dataset}_dataset.h5\")\n",
"\n",
"# USER: specify the path to the parameter file\n",
"\n",
"parfile = '/home/esrf/james1997a/Data/ihma439/id11/20231211/PROCESSED_DATA/James/20240226/Fe_tdxrd_refined.par'\n",
"spline_file = '/data/id11/inhouse1/ewoks/detectors/files/Frelon2k_C36/frelon36.spline'"
"dset_path = os.path.join(processed_data_root_dir, sample, f\"{sample}_{dataset}\", f\"{sample}_{dataset}_dataset.h5\")"
]
},
{
Expand Down Expand Up @@ -159,7 +154,7 @@
"\n",
"cf_3d = ImageD11.columnfile.colfile_from_hdf(ds.col3dfile)\n",
"\n",
"cf_3d.parameters.loadparameters(parfile)\n",
"cf_3d.parameters.loadparameters(ds.parfile)\n",
"cf_3d.updateGeometry()\n",
"\n",
"\n",
Expand Down Expand Up @@ -203,7 +198,7 @@
"# this indicates the fractional intensity cutoff we will select\n",
"# if the blue line does not look elbow-shaped in the logscale plot, try changing the \"doplot\" parameter (the y scale of the logscale plot) until it does\n",
"\n",
"cf_strong = utils.selectpeaks(cf_3d, frac=0.985, dsmax=0.92, doplot=0.8, dstol=0.01)\n",
"cf_strong = utils.selectpeaks(cf_3d, frac=0.985, dsmax=1.04, doplot=0.8, dstol=0.01)\n",
"print(f\"Got {cf_strong.nrows} strong peaks for indexing\")\n",
"cf_strong.writefile(f'{sample}_{dataset}_3d_peaks_strong.flt')"
]
Expand Down Expand Up @@ -321,23 +316,28 @@
"outputs": [],
"source": [
"# now we are indexing!\n",
"# indexing will select all rings with a multiplicity below max_multiplity to search\n",
"max_multiplicity = 13\n",
"# the minimum number of peaks on a ring for a ring to be indexed on\n",
"min_counts_on_ring = 0\n",
"# we have to choose which rings we want to generate orientations on\n",
"# generally we want two or three low-multiplicity rings that are isolated from other phases\n",
"# take a look at the ring assignment output from a few cells above, and choose two or three\n",
"rings_for_gen = [0, 1]\n",
"\n",
"# now we want to decide which rings to score our found orientations against\n",
"# generally we can just exclude dodgy rings (close to other phases, only a few peaks in etc)\n",
"rings_for_scoring = [0, 1, 2, 3]\n",
"\n",
"# the sequence of hkl tolerances the indexer will iterate through\n",
"hkl_tols_seq = [0.01, 0.02, 0.03, 0.04]\n",
"# the sequence of minpks fractions the indexer will iterate through\n",
"fracs = [0.9, 0.75]\n",
"# the tolerance in g-vector angle\n",
"cosine_tol = np.cos(np.radians(90.25))\n",
"cosine_tol = np.cos(np.radians(90 - 0.25))\n",
"# the max number of UBIs we can find per pair of rings\n",
"max_grains = 1000\n",
"\n",
"grains, indexer = utils.do_index(cf=cf_strong,\n",
" dstol=indexer.ds_tol,\n",
" max_mult=max_multiplicity,\n",
" min_ring_count=min_counts_on_ring,\n",
" forgen=rings_for_gen,\n",
" foridx=rings_for_scoring,\n",
" hkl_tols=hkl_tols_seq,\n",
" fracs=fracs,\n",
" cosine_tol=cosine_tol,\n",
Expand Down Expand Up @@ -412,9 +412,9 @@
"for inc, makemap_tol in enumerate(makemap_hkl_tol_seq):\n",
" print(f\"Running makemap {inc+1}/{len(makemap_hkl_tol_seq)}\")\n",
" if inc == 0: # ubi into map\n",
" makemap_output = !makemap.py -p {parfile} -u {tmp_ubi_path} -U {tmp_map_path} -f {cf_strong_allrings_path} -F {unindexed_flt_path} -s cubic -t {makemap_hkl_tol_seq[inc]} --omega_slop={omega_slop} --no_sort\n",
" makemap_output = !makemap.py -p {ds.parfile} -u {tmp_ubi_path} -U {tmp_map_path} -f {cf_strong_allrings_path} -F {unindexed_flt_path} -s cubic -t {makemap_hkl_tol_seq[inc]} --omega_slop={omega_slop} --no_sort\n",
" else: # map into map\n",
" makemap_output = !makemap.py -p {parfile} -u {tmp_map_path} -U {tmp_map_path} -f {cf_strong_allrings_path} -F {unindexed_flt_path} -s cubic -t {makemap_hkl_tol_seq[inc]} --omega_slop={omega_slop} --no_sort"
" makemap_output = !makemap.py -p {ds.parfile} -u {tmp_map_path} -U {tmp_map_path} -f {cf_strong_allrings_path} -F {unindexed_flt_path} -s cubic -t {makemap_hkl_tol_seq[inc]} --omega_slop={omega_slop} --no_sort"
]
},
{
Expand Down Expand Up @@ -449,7 +449,7 @@
},
"outputs": [],
"source": [
"refine_output = !refine_em.py {new_flt_path} {tmp_map_path} {parfile} --omega_slop={omega_slop}"
"refine_output = !refine_em.py {new_flt_path} {tmp_map_path} {ds.parfile} --omega_slop={omega_slop}"
]
},
{
Expand Down
Loading

0 comments on commit 8ae3999

Please sign in to comment.