Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,7 @@ RUN \
cargo install cargo-llvm-cov && \
rustup component add llvm-tools-preview && \
# rust crate structure diagram
cargo install cargo-modules
cargo install cargo-modules && \
# expand rust macros (useful in debugging)
cargo install cargo-expand
CMD ["bash", "-c", "sudo rm /var/run/docker.pid; sudo dockerd"]
2 changes: 2 additions & 0 deletions .devcontainer/gpu/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ RUN apt-get update && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs
rustup component add llvm-tools-preview && \
# rust crate structure diagram
cargo install cargo-modules && \
# expand rust macros (useful in debugging)
cargo install cargo-expand && \
apt-get clean

ENV PATH=${PATH}:/root/.local/bin
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/docs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,11 @@ jobs:
- name: Generate crate diagram
run: |
mkdir -p docs
set -o pipefail
cargo modules dependencies \
--no-uses --no-fns \
--focus-on "orcapod::model::{Pod}" \
--focus-on "orcapod::uniffi::model::{Pod}" \
--layout dot | \
awk '/ERROR/ { print > "/dev/stderr"; next; }; 1' | \
dot -T svg > docs/crate_diagram.svg
- name: Sync GitHub
run: |
Expand Down
16 changes: 4 additions & 12 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -72,13 +72,9 @@
"cargo": {
"args": [
"build",
"--bin=exe_file_stem",
"--package=orcapod"
"--package=orcapod",
"--bin=exe_file_stem"
],
"filter": {
"name": "exe_file_stem",
"kind": "bin"
}
},
"args": [],
"cwd": "${workspaceFolder}",
Expand All @@ -91,13 +87,9 @@
"args": [
"test",
"--no-run",
"--bin=exe_file_stem",
"--package=orcapod"
"--package=orcapod",
"--bin=exe_file_stem"
],
"filter": {
"name": "exe_file_stem",
"kind": "bin"
}
},
"args": [],
"cwd": "${workspaceFolder}",
Expand Down
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ serde = { version = "1.0.210", features = ["derive"] } # serialization/deseriali
serde_json = "1.0.137" # JSON in sharing memory with local docker orchestrator
serde_yaml = "0.9.34" # YAML in filestore
sha2 = "0.10.8" # checksums based on SHA256
thiserror = "2.0.11" # library error handling API
snafu = { version = "0.8.5", features = ["futures"] } # library error handling API
tokio = { version = "1.41.0", features = ["full"] } # a runtime for async applications
tokio-util = "0.7.13" # utilities for async calls

Expand Down
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ cargo llvm-cov --html -- --nocapture # integration tests w/ coverage report

```bash
cargo doc --no-deps # gen api docs (target/doc/orcapod/index.html)
cargo modules dependencies --no-uses --no-fns --focus-on "orcapod::model::{Pod}" --layout dot > docs/crate_diagram.dot # orcapod diagram as DOT
cargo modules dependencies --no-uses --no-fns --focus-on "orcapod::model::{Pod}" --layout dot | dot -T png > docs/crate_diagram.png # orcapod diagram as PNG
cargo modules dependencies --no-uses --no-fns --focus-on "orcapod::model::{Pod}" --layout dot | dot -T svg > docs/crate_diagram.svg # orcapod diagram as SVG
cargo modules dependencies --no-uses --no-fns --focus-on "orcapod::uniffi::model::{Pod}" --layout dot > docs/crate_diagram.dot # orcapod diagram as DOT
cargo modules dependencies --no-uses --no-fns --focus-on "orcapod::uniffi::model::{Pod}" --layout dot | dot -T png > docs/crate_diagram.png # orcapod diagram as PNG
cargo modules dependencies --no-uses --no-fns --focus-on "orcapod::uniffi::model::{Pod}" --layout dot | dot -T svg > docs/crate_diagram.svg # orcapod diagram as SVG
```

## Project Management
Expand Down
9 changes: 7 additions & 2 deletions src/core/crypto.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
use crate::{
core::util::get,
uniffi::{
error::Result,
error::{Result, selector},
model::{Blob, BlobKind},
},
};
use serde_yaml;
use sha2::{Digest as _, Sha256};
use snafu::ResultExt as _;
use std::{
collections::{BTreeMap, HashMap},
fs::File,
Expand Down Expand Up @@ -47,7 +48,11 @@ pub fn hash_buffer(buffer: impl AsRef<[u8]>) -> String {
///
/// Will return error if unable to access file.
pub fn hash_file(filepath: impl AsRef<Path>) -> Result<String> {
hash_stream(&mut File::open(filepath)?)
hash_stream(
&mut File::open(&filepath).context(selector::InvalidFilepath {
path: filepath.as_ref(),
})?,
)
}
/// Evaluate checksum hash of a directory.
///
Expand Down
89 changes: 61 additions & 28 deletions src/core/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,59 +4,92 @@ use glob;
use serde_json;
use serde_yaml;
use std::{
fmt::{self, Display, Formatter},
io, path,
backtrace::{Backtrace, BacktraceStatus},
fmt::{self, Formatter},
io,
path::{self},
};

impl Display for OrcaError {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.kind)
}
}
impl From<BollardError> for OrcaError {
fn from(error: BollardError) -> Self {
Self {
kind: Kind::BollardError(error),
}
Self(Kind::BollardError {
source: error,
backtrace: Some(Backtrace::capture()),
})
}
}
impl From<glob::PatternError> for OrcaError {
fn from(error: glob::PatternError) -> Self {
Self {
kind: Kind::GlobPatternError(error),
}
Self(Kind::GlobPatternError {
source: error,
backtrace: Some(Backtrace::capture()),
})
}
}
impl From<io::Error> for OrcaError {
fn from(error: io::Error) -> Self {
Self {
kind: Kind::IoError(error),
}
Self(Kind::IoError {
source: error,
backtrace: Some(Backtrace::capture()),
})
}
}
impl From<path::StripPrefixError> for OrcaError {
fn from(error: path::StripPrefixError) -> Self {
Self {
kind: Kind::PathPrefixError(error),
}
Self(Kind::PathPrefixError {
source: error,
backtrace: Some(Backtrace::capture()),
})
}
}
impl From<serde_json::Error> for OrcaError {
fn from(error: serde_json::Error) -> Self {
Self {
kind: Kind::SerdeJsonError(error),
}
Self(Kind::SerdeJsonError {
source: error,
backtrace: Some(Backtrace::capture()),
})
}
}
impl From<serde_yaml::Error> for OrcaError {
fn from(error: serde_yaml::Error) -> Self {
Self {
kind: Kind::SerdeYamlError(error),
}
Self(Kind::SerdeYamlError {
source: error,
backtrace: Some(Backtrace::capture()),
})
}
}
impl From<Kind> for OrcaError {
fn from(kind: Kind) -> Self {
Self { kind }
fn format_stack(backtrace: Option<&Backtrace>) -> String {
backtrace.map_or(
String::new(),
|unpacked_backtrace| match unpacked_backtrace.status() {
BacktraceStatus::Captured => {
format!("\nstack backtrace:\n{unpacked_backtrace}")
}
BacktraceStatus::Disabled | BacktraceStatus::Unsupported | _ => String::new(),
},
)
}
impl fmt::Debug for OrcaError {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
match &self.0 {
Kind::EmptyResponseWhenLoadingContainerAltImage { backtrace, .. }
| Kind::GeneratedNamesOverflow { backtrace, .. }
| Kind::InvalidFilepath { backtrace, .. }
| Kind::InvalidPodResultTerminatedDatetime { backtrace, .. }
| Kind::KeyMissing { backtrace, .. }
| Kind::NoAnnotationFound { backtrace, .. }
| Kind::NoContainerNames { backtrace, .. }
| Kind::NoFileName { backtrace, .. }
| Kind::NoMatchingPodRun { backtrace, .. }
| Kind::NoTagFoundInContainerAltImage { backtrace, .. }
| Kind::BollardError { backtrace, .. }
| Kind::GlobPatternError { backtrace, .. }
| Kind::IoError { backtrace, .. }
| Kind::PathPrefixError { backtrace, .. }
| Kind::SerdeJsonError { backtrace, .. }
| Kind::SerdeYamlError { backtrace, .. } => {
write!(f, "{}{}", self.0, format_stack(backtrace.as_ref()))
}
}
}
}
13 changes: 7 additions & 6 deletions src/core/orchestrator/docker.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use crate::{
core::util::get,
uniffi::{
error::{Kind, OrcaError, Result},
error::{Result, selector},
model::{Input, PodJob},
orchestrator::{RunInfo, Status, docker::LocalDockerOrchestrator},
},
Expand All @@ -14,6 +14,7 @@ use chrono::DateTime;
use futures_util::future::join_all;
use names::{Generator, Name};
use regex::Regex;
use snafu::OptionExt as _;
use std::{
collections::HashMap,
fs,
Expand Down Expand Up @@ -81,9 +82,9 @@ impl LocalDockerOrchestrator {
.to_string_lossy(),
stream_info
.path
.join(blob.location.path.file_name().ok_or(
Kind::InvalidPath {
path: blob.location.path.clone(),
.join(blob.location.path.file_name().context(
selector::NoFileName {
path: blob.location.path.clone()
}
)?)
.to_string_lossy(),
Expand Down Expand Up @@ -120,7 +121,7 @@ impl LocalDockerOrchestrator {
let (input_binds, output_bind) = Self::prepare_mount_binds(namespace_lookup, pod_job)?;
let container_name = Generator::with_naming(Name::Plain)
.next()
.ok_or(OrcaError::from(Kind::GeneratedNamesOverflow))?;
.context(selector::GeneratedNamesOverflow)?;
let labels = HashMap::from([
("org.orcapod".to_owned(), "true".to_owned()),
(
Expand Down Expand Up @@ -207,7 +208,7 @@ impl LocalDockerOrchestrator {
let container_name = &container_summary
.names
.as_ref()
.ok_or(OrcaError::from(Kind::NoContainerNames))?[0][1..];
.context(selector::NoContainerNames)?[0][1..];
Ok((
container_name.to_owned(),
container_summary.clone(),
Expand Down
37 changes: 22 additions & 15 deletions src/core/store/filestore.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use crate::{
core::{model::to_yaml, util::get_type_name},
uniffi::{
error::{Kind, OrcaError, Result},
error::{Result, selector},
model::Annotation,
store::{ModelID, ModelInfo, Store as _, filestore::LocalFileStore},
},
Expand All @@ -12,6 +12,7 @@ use heck::ToSnakeCase as _;
use regex::Regex;
use serde::{Serialize, de::DeserializeOwned};
use serde_yaml;
use snafu::OptionExt as _;
use std::{
fs,
path::{Path, PathBuf},
Expand Down Expand Up @@ -85,12 +86,10 @@ impl LocalFileStore {
&self.make_path::<T>("*", Self::make_annotation_relpath(name, version)),
)?
.next()
.ok_or_else(|| {
OrcaError::from(Kind::NoAnnotationFound {
class: get_type_name::<T>().to_snake_case(),
name: name.to_owned(),
version: version.to_owned(),
})
.context(selector::NoAnnotationFound {
class: get_type_name::<T>().to_snake_case(),
name: name.to_owned(),
version: version.to_owned(),
})?;
Ok(model_info.hash)
}
Expand Down Expand Up @@ -128,11 +127,15 @@ impl LocalFileStore {
})
{
println!(
"Skip saving {} annotation since `{}`, `{}`, `{}` exists.",
model_type.bright_cyan(),
found_hash.bright_cyan(),
found_name.bright_cyan(),
found_version.bright_cyan(),
"{}",
format!(
"Skip saving {} annotation since `{}`, `{}`, `{}` exists.",
model_type.bright_cyan(),
found_hash.bright_cyan(),
found_name.bright_cyan(),
found_version.bright_cyan(),
)
.yellow(),
);
} else {
Self::save_file(
Expand All @@ -145,9 +148,13 @@ impl LocalFileStore {
let spec_file = &self.make_path::<T>(hash, Self::SPEC_RELPATH);
if spec_file.exists() {
println!(
"Skip saving {} model since `{}` exists.",
model_type.bright_cyan(),
hash.bright_cyan(),
"{}",
format!(
"Skip saving {} model since `{}` exists.",
model_type.bright_cyan(),
hash.bright_cyan(),
)
.yellow(),
);
} else {
Self::save_file(spec_file, to_yaml(model)?)?;
Expand Down
5 changes: 3 additions & 2 deletions src/core/util.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use crate::uniffi::error::{Kind, Result};
use crate::uniffi::error::{Result, selector};
use snafu::OptionExt as _;
use std::{any::type_name, collections::HashMap};

#[expect(
Expand All @@ -14,7 +15,7 @@ pub fn get_type_name<T>() -> String {
}

pub fn get<'map, T>(map: &'map HashMap<String, T>, key: &str) -> Result<&'map T> {
Ok(map.get(key).ok_or(Kind::KeyMissing {
Ok(map.get(key).context(selector::KeyMissing {
key: key.to_owned(),
})?)
}
Loading