Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 31 additions & 3 deletions ifcbdb/assets/js/bin.js
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,6 @@ function updateBinDownloadLinks(data) {
$("#download-zip").attr("href", infix + _bin + ".zip");
$("#download-blobs").attr("href", infix + _bin + "_blob.zip");
$("#download-features").attr("href", infix + _bin + "_features.csv");
$("#download-class-scores").attr("href", infix + _bin + "_class_scores.csv");

$.get('/api/has_products/' + _bin, function(r) {
$("#download-blobs").toggle(r["has_blobs"]);
Expand All @@ -261,8 +260,37 @@ function updateBinDownloadLinks(data) {
$("#download-features").toggle(r["has_features"]);
$("#download-features-disabled").toggle(!r["has_features"]);

$("#download-class-scores").toggle(r["has_class_scores"]);
$("#download-class-scores-disabled").toggle(!r["has_class_scores"]);
// Remove existing class scores, including the disable placeholder
$("#features-list li.class-score").remove();

const classScores = r.class_scores.map((item) => {
const href = infix + _bin + "_class_scores.csv?model=" + item.model;
const text = "autoclass" + (item.model ? ` (${item.model})` : "");

return $("<a />", {
text: text,
href: href
});
});

// If there are no class scores, show a placeholder w/o a link
if (!classScores.length) {
const placeholder = $("<span />", {
class: "download-class-scores-disabled",
text: "autoclass"
});

classScores.push(placeholder);
}

classScores.forEach((item) => {
const listItem = $("<li />", {
class: "class-score",
html: item
});

$("#features-list").append(listItem);
});

// Update outline/blob links
$("#detailed-image-blob-link").toggleClass("disabled", !r["has_blobs"]);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Generated by Django 4.2.29 on 2026-04-10 03:10

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('dashboard', '0052_team_short_description'),
]

operations = [
migrations.AddField(
model_name='datadirectory',
name='is_class_score_default',
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name='datadirectory',
name='model',
field=models.SlugField(blank=True, max_length=100, null=True),
),
]
18 changes: 18 additions & 0 deletions ifcbdb/dashboard/migrations/0054_alter_datadirectory_model.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Generated by Django 4.2.30 on 2026-04-21 03:18

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('dashboard', '0053_datadirectory_is_class_score_default_and_more'),
]

operations = [
migrations.AlterField(
model_name='datadirectory',
name='model',
field=models.SlugField(blank=True, default='', max_length=100),
),
]
50 changes: 38 additions & 12 deletions ifcbdb/dashboard/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,9 @@ class DataDirectory(models.Model):
blacklist = models.CharField(max_length=512, default='skip,bad') # comma separated list of directory names to skip
# for product directories, the product version
version = models.IntegerField(null=True, blank=True)
model = models.SlugField(max_length=100, blank=True, null=False, default="")
# for class score directories; the default directory if there is more than one (falls back to most recently created)
is_class_score_default = models.BooleanField(default=False, blank=False, null=False)

def get_raw_directory(self):
if self.kind != self.RAW:
Expand Down Expand Up @@ -607,12 +610,24 @@ def set_ml_analyzed(self, ml_analyzed):

# access to underlying FilesetBin objects

def _directories(self, kind=DataDirectory.RAW, version=None):
def _directories(self, kind=DataDirectory.RAW, version=None, model=None):
for dataset in self.datasets.all():
qs = dataset.directories.filter(kind=kind)
if version is not None:
qs = qs.filter(version=version)
for directory in qs.order_by('priority'):

# A value of None for the model indicates it is not specified and should not be filtered
# on. But model could also be an empty string, in which case this should return that
# specific data directory (the one w/o a model set on it)
if model is not None:
qs = qs.filter(model=model)

if kind == DataDirectory.CLASS_SCORES:
qs = qs.order_by("-is_class_score_default", "-pk")
else:
qs = qs.order_by("priority")

for directory in qs:
yield directory

def _get_bin(self):
Expand Down Expand Up @@ -731,27 +746,38 @@ def features(self, version=None):

# class scores

def class_scores_file(self, version=None):
def class_scores_file_list(self, version=None):
class_scores = []

for directory in self._directories(kind=DataDirectory.CLASS_SCORES, version=version):
csd = directory.get_class_scores_directory()
try:
class_scores.append(
{
"path": csd[self.pid].path,
"model": directory.model,
}
)
except KeyError:
pass

return class_scores

def class_scores_file(self, version=None, model=None):
for directory in self._directories(kind=DataDirectory.CLASS_SCORES, version=version, model=model):
csd = directory.get_class_scores_directory()
try:
return csd[self.pid]
except KeyError:
pass
raise KeyError('no class scores found for {}'.format(self.pid))

def has_class_scores(self, version=None):
try:
self.class_scores_file(version=version)
return True
except KeyError:
return False
raise KeyError("no class scores found for {}".format(self.pid))

def class_scores_path(self, version=None):
return self.class_scores_file(version=version).path

def class_scores(self, version=None):
return self.class_scores_file(version=version).class_scores()
def class_scores(self, version=None, model=None):
return self.class_scores_file(version=version, model=model).class_scores()

# mosaics

Expand Down
20 changes: 14 additions & 6 deletions ifcbdb/dashboard/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -750,28 +750,37 @@ def class_scores_mat(request, bin_id, **kw):
if 'dataset_name' in kw:
bin_in_dataset_or_404(b, kw['dataset_name'])
version = get_product_version_parameter(request)
model = request.GET.get("model")

try:
class_scores_file = b.class_scores_file(version=version)
version = class_scores_file.version
class_scores_path = class_scores_file.path
except KeyError:
raise Http404
filename = '{}_class_v{}.mat'.format(bin_id, version)

filename = bin_id + ("_" + model if model is not None else "") + ".mat"

fin = open(class_scores_path, 'rb')
return FileResponse(fin, as_attachment=True, filename=filename, content_type='application/octet-stream')

def class_scores_csv(request, dataset_name, bin_id):
b = get_object_or_404(Bin, pid=bin_id)
bin_in_dataset_or_404(b, dataset_name)
version = get_product_version_parameter(request, None)
model = request.GET.get("model")

try:
class_scores = b.class_scores(version=version)
class_scores = b.class_scores(version=version, model=model)
except KeyError:
raise Http404

class_scores.index = ['{}_{:05d}'.format(bin_id, tn) for tn in class_scores.index]
class_scores.index.name = 'pid'
resp = dataframe_csv_response(class_scores)
filename = '{}_class_v{}.csv'.format(bin_id, version)

filename = bin_id + ("_" + model if model is not None else "") + ".csv"

resp['Content-Disposition'] = 'attachment; filename={}'.format(filename)
return resp

Expand Down Expand Up @@ -861,10 +870,8 @@ def _bin_details(bin, dataset=None, view_size=None, scale_factor=None, preload_a
"coordinates": coordinates_json,
#"has_blobs": bin.has_blobs(),
#"has_features": bin.has_features(),
#"has_class_scores": bin.has_class_scores(), # FIXME slow
"has_blobs": False,
"has_features": False,
"has_class_scores": False,
"timestamp_iso": bin.sample_time.isoformat(),
"instrument": "IFCB" + str(bin.instrument.number),
"num_triggers": bin.n_triggers,
Expand Down Expand Up @@ -1210,11 +1217,12 @@ def filter_options(request):

def has_products(request, bin_id):
b = get_object_or_404(Bin, pid=bin_id)
class_scores = b.class_scores_file_list()

return JsonResponse({
"has_blobs": b.has_blobs(),
"has_features": b.has_features(),
"has_class_scores": b.has_class_scores(),
"class_scores": class_scores,
})

# legacy feed view
Expand Down
28 changes: 25 additions & 3 deletions ifcbdb/secure/forms.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,22 +142,43 @@ def clean(self):
data = self.cleaned_data
path = self.cleaned_data.get("path")
kind = self.cleaned_data.get("kind")
instance_id = self.instance.id if self.instance else 0

# make sure the directory path is not already in the database
existing_path = DataDirectory.objects.filter(dataset_id=self.dataset_id, path=path, kind=kind).first()
if existing_path:
existing_path = DataDirectory.objects \
.filter(dataset_id=self.dataset_id, path=path, kind=kind) \
.exclude(id=instance_id)

if existing_path.exists():
raise forms.ValidationError({
'path': 'Path "{}" (kind: {}) is already in use'.format(path, kind)
})

# Class score directories have an additional requirement to not allow for duplicate model values. This
# includes preventing more than one directory where the model value is left blank
if kind == DataDirectory.CLASS_SCORES:
model = self.cleaned_data.get("model") or ""

existing_model = DataDirectory.objects \
.filter(dataset_id=self.dataset_id, kind=kind, model=model) \
.exclude(id=instance_id)

if existing_model.exists():
msg = f"Model {model} is already in use" if model \
else "Only one class score data directory can be created with a blank value for the model"

raise forms.ValidationError({
'path': msg
})

return data

def _match_folder_names(self, value):
return re.match(r'^[A-Za-z0-9,\s]*$', value)

class Meta:
model = DataDirectory
fields = ["id", "path", "kind", "priority", "whitelist", "blacklist", "version", ]
fields = ["id", "path", "kind", "priority", "whitelist", "blacklist", "version", "model", "is_class_score_default" ]

widgets = {
"path": forms.TextInput(attrs={"class": "form-control form-control-sm", "placeholder": "Path"}),
Expand All @@ -173,6 +194,7 @@ class Meta:
"blacklist": forms.TextInput(attrs={"class": "form-control form-control-sm", "placeholder": "Blacklist"}),
"version": forms.TextInput(attrs={"class": "form-control form-control-sm", "placeholder": "Version"}),
"priority": forms.TextInput(attrs={"class": "form-control form-control-sm", "placeholder": "Priority"}),
"model": forms.TextInput(attrs={"class": "form-control form-control-sm", "placeholder": "Model"}),
}


Expand Down
5 changes: 2 additions & 3 deletions ifcbdb/templates/dashboard/bin.html
Original file line number Diff line number Diff line change
Expand Up @@ -578,7 +578,7 @@
</ul>
</div>
<div class="flex-column flex-fill">
<ul class="list-unstyled">
<ul class="list-unstyled" id="features-list">
<li>
<a id="download-blobs" href="#" style="display:none">blobs</a>
<span id="download-blobs-disabled">blobs</span>
Expand All @@ -587,8 +587,7 @@
<a id="download-features" href="#" style="display:none">features</a>
<span id="download-features-disabled">features</span>
</li>
<li>
<a id="download-class-scores" href="#" style="display:none">autoclass</a>
<li class="class-score">
<span id="download-class-scores-disabled">autoclass</span>
</li>
</ul>
Expand Down
27 changes: 23 additions & 4 deletions ifcbdb/templates/secure/edit-directory.html
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,22 @@
{% if form.version.errors %}<span class="text-danger">{{ form.version.errors.as_text }}</span>{% endif %}
</div>
</div>
<div id="model-container" class="form-row d-none">
<div class="form-group col">
<label for="id_model">Model</label>
{{ form.model }}
{% if form.model.errors %}<span class="text-danger">{{ form.model.errors.as_text }}</span>{% endif %}
</div>
</div>
<div id="is-class-score-default-container" class="form-row d-none">
<div class="form-group col">
<label for="id_is_class_score_default">Default Class Score Data Directory?</label>
<div>
{{ form.is_class_score_default }}
</div>
{% if form.is_class_score_default.errors %}<span class="text-danger">{{ form.is_class_score_default.errors.as_text }}</span>{% endif %}
</div>
</div>
<div class="form-row">
<div class="form-group col">
<label for="id_priority">Priority</label>
Expand All @@ -74,16 +90,19 @@
{% block scripts %}
<script>
function updateKind() {
console.log("ASD");
var kind = $("#id_kind").val();
$("#version-container").toggleClass("d-none", kind == "raw");
const kind = $("#id_kind").val();

$("#version-container").toggleClass("d-none", kind === "raw");
$("#model-container").toggleClass("d-none", kind !== "class_scores");
$("#is-class-score-default-container").toggleClass("d-none", kind !== "class_scores");
}

$(function(){
updateKind();

$("#id_kind").change(function(e){
updateKind();
});
});

</script>
{% endblock %}