diff --git a/ifcbdb/dashboard/accession.py b/ifcbdb/dashboard/accession.py
index 9efd30e8..a76a7635 100644
--- a/ifcbdb/dashboard/accession.py
+++ b/ifcbdb/dashboard/accession.py
@@ -1,3 +1,4 @@
+from pprint import pprint
import os
import json
import time
@@ -59,21 +60,27 @@ def scan(self):
for dd in self.dataset.directories.filter(kind=DataDirectory.RAW).order_by('priority'):
if not os.path.exists(dd.path):
continue # skip and continue searching
- directory = ifcb.DataDirectory(dd.path)
+
+ directory = ifcb.DataDirectory(dd.path, require_roi_files=dd.require_roi_files)
for b in directory:
+ # Bins do not have a require_roi_files property, but we're adding it here anyway to pass
+ # along the associated directory's setting through to the next call
+ b.require_roi_files = dd.require_roi_files
yield b
def sync_one(self, pid):
bin = None
+ directory = None
for dd in self.dataset.directories.filter(kind=DataDirectory.RAW).order_by('priority'):
if not os.path.exists(dd.path):
continue # skip and continue searching
- directory = ifcb.DataDirectory(dd.path)
+ directory = ifcb.DataDirectory(dd.path, require_roi_files=dd.require_roi_files)
try:
bin = directory[pid]
except KeyError:
continue
if bin is None:
return 'bin {} not found'.format(pid)
+
# create instrument if necessary
i = bin.pid.instrument
version = bin.pid.schema_version
@@ -161,6 +168,7 @@ def sync(self, progress_callback=do_nothing, log_callback=do_nothing):
bad_bins += 1
else:
log_callback('{} not adding bin'.format(b.pid))
+
with transaction.atomic():
for b in bins2save:
b.skip = False # unskip because we're ready to save
@@ -181,13 +189,13 @@ def sync(self, progress_callback=do_nothing, log_callback=do_nothing):
progress_callback(prog)
return prog
- def add_bin(self, bin, b): # IFCB bin, Bin instance
+ def add_bin(self, bin, b, require_roi_files=True): # IFCB bin, Bin instance
# qaqc checks
qc_bad = check_bad(bin)
if qc_bad:
b.qc_bad = True
return b, 'malformed raw data'
- no_rois = check_no_rois(bin)
+ no_rois = require_roi_files and check_no_rois(bin)
if no_rois:
b.qc_bad = True
return b, 'zero ROIs'
@@ -227,7 +235,7 @@ def add_bin(self, bin, b): # IFCB bin, Bin instance
if latitude is not None and longitude is not None:
b.set_location(longitude, latitude, depth)
#
- b.qc_no_rois = check_no_rois(bin)
+ b.qc_no_rois = require_roi_files and check_no_rois(bin)
# metrics
try:
b.temperature = bin.temperature
diff --git a/ifcbdb/dashboard/migrations/0039_datadirectory_require_roi_files.py b/ifcbdb/dashboard/migrations/0039_datadirectory_require_roi_files.py
new file mode 100644
index 00000000..35bf25c3
--- /dev/null
+++ b/ifcbdb/dashboard/migrations/0039_datadirectory_require_roi_files.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.20 on 2025-04-17 20:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('dashboard', '0038_remove_datadirectory_unique_path_and_more'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='datadirectory',
+ name='require_roi_files',
+ field=models.BooleanField(default=True),
+ ),
+ ]
diff --git a/ifcbdb/dashboard/models.py b/ifcbdb/dashboard/models.py
index 6a2c7655..8c992e9e 100644
--- a/ifcbdb/dashboard/models.py
+++ b/ifcbdb/dashboard/models.py
@@ -368,6 +368,7 @@ class DataDirectory(models.Model):
blacklist = models.CharField(max_length=512, default='skip,bad') # comma separated list of directory names to skip
# for product directories, the product version
version = models.IntegerField(null=True, blank=True)
+ require_roi_files = models.BooleanField(default=True)
def get_raw_directory(self):
if self.kind != self.RAW:
@@ -375,7 +376,7 @@ def get_raw_directory(self):
# return the underlying ifcb.DataDirectory
whitelist = re.split(',', self.whitelist)
blacklist = re.split(',', self.blacklist)
- return ifcb.DataDirectory(self.path, whitelist=whitelist, blacklist=blacklist)
+ return ifcb.DataDirectory(self.path, whitelist=whitelist, blacklist=blacklist, require_roi_files=self.require_roi_files)
def raw_destination(self, bin_id):
# where to put an incoming bin with the given id
diff --git a/ifcbdb/secure/forms.py b/ifcbdb/secure/forms.py
index 866ab6d3..00f07a15 100644
--- a/ifcbdb/secure/forms.py
+++ b/ifcbdb/secure/forms.py
@@ -128,7 +128,7 @@ def _match_folder_names(self, value):
class Meta:
model = DataDirectory
- fields = ["id", "path", "kind", "priority", "whitelist", "blacklist", "version", ]
+ fields = ["id", "path", "kind", "priority", "whitelist", "blacklist", "version", "require_roi_files", ]
widgets = {
"path": forms.TextInput(attrs={"class": "form-control form-control-sm", "placeholder": "Path"}),
@@ -144,6 +144,7 @@ class Meta:
"blacklist": forms.TextInput(attrs={"class": "form-control form-control-sm", "placeholder": "Blacklist"}),
"version": forms.TextInput(attrs={"class": "form-control form-control-sm", "placeholder": "Version"}),
"priority": forms.TextInput(attrs={"class": "form-control form-control-sm", "placeholder": "Priority"}),
+ "require_roi_files": forms.CheckboxInput(attrs={"class": "custom-control-input"})
}
diff --git a/ifcbdb/secure/views.py b/ifcbdb/secure/views.py
index c6a13493..11f883c7 100644
--- a/ifcbdb/secure/views.py
+++ b/ifcbdb/secure/views.py
@@ -59,7 +59,7 @@ def dt_datasets(request):
@login_required
def dt_directories(request, dataset_id):
directories = list(DataDirectory.objects.filter(dataset__id=dataset_id)
- .values_list("path", "kind", "priority", "whitelist", "blacklist", "id"))
+ .values_list("path", "kind", "priority", "whitelist", "blacklist", "require_roi_files", "id"))
return JsonResponse({
"data": directories,
diff --git a/ifcbdb/templates/secure/directory-management.html b/ifcbdb/templates/secure/directory-management.html
index c1a6a9b0..c8e58bad 100644
--- a/ifcbdb/templates/secure/directory-management.html
+++ b/ifcbdb/templates/secure/directory-management.html
@@ -22,6 +22,7 @@
| Priority |
Whitelist |
Blacklist |
+ Require ROI Files |
|
@@ -75,6 +76,9 @@ Confirm Removal
{}, // Priority
{}, // Whitelist
{}, // Blacklist
+ {
+ render: function(data) { return data ? "Yes" : "No" }
+ }, // Require ROI Files
{ // Edit
targets: -1,
render: function ( data, type, row ) {
diff --git a/ifcbdb/templates/secure/edit-directory.html b/ifcbdb/templates/secure/edit-directory.html
index 5b47c254..ff727625 100644
--- a/ifcbdb/templates/secure/edit-directory.html
+++ b/ifcbdb/templates/secure/edit-directory.html
@@ -64,6 +64,15 @@
{% if form.priority.errors %}{{ form.priority.errors.as_text }}{% endif %}
+