cirro.services

10class BillingService(BaseService):
11    """
12    Service for interacting with the Billing endpoints
13    """
14    def list(self) -> List[BillingAccount]:
15        """
16        Gets a list of billing accounts the current user has access to
17        """
18        return get_billing_accounts.sync(client=self._api_client)
19
20    def update(self, billing_account_id: str, request: BillingAccountRequest):
21        """
22        Updates a billing account
23
24        Args:
25            billing_account_id (str): Billing account ID
26            request (cirro_api_client.v1.models.BillingAccountRequest):
27
28        ```python
29        from cirro_api_client.v1.models import BillingAccountRequest
30        from cirro.cirro_client import CirroApi
31
32        cirro = CirroApi()
33        request = BillingAccountRequest(
34            name="New billing account name",
35            primary_budget_number="new-budget-number",
36            owner="New Owner"
37        )
38        cirro.billing.update("billing-account-id", request)
39        ```
40        """
41        update_billing_account.sync_detailed(
42            billing_account_id=billing_account_id,
43            body=request,
44            client=self._api_client
45        )

 14class DatasetService(FileEnabledService):
 15    """
 16    Service for interacting with the Dataset endpoints
 17    """
 18    def list(self, project_id: str, max_items: int = 10000) -> List[Dataset]:
 19        """List datasets
 20
 21        Retrieves a list of datasets for a given project
 22
 23        Args:
 24            project_id (str): ID of the Project
 25            max_items (int): Maximum number of records to get (default 10,000)
 26        """
 27        return get_all_records(
 28            records_getter=lambda page_args: get_datasets.sync(
 29                project_id=project_id,
 30                client=self._api_client,
 31                next_token=page_args.next_token,
 32                limit=page_args.limit
 33            ),
 34            max_items=max_items
 35        )
 36
 37    def import_public(self, project_id: str, import_request: ImportDataRequest) -> CreateResponse:
 38        """
 39        Download data from public repositories
 40
 41        Args:
 42            project_id (str): ID of the Project
 43            import_request (cirro_api_client.v1.models.ImportDataRequest):
 44
 45        Returns:
 46            ID of the created dataset
 47        """
 48        return import_public_dataset.sync(project_id=project_id, client=self._api_client, body=import_request)
 49
 50    def create(self, project_id: str, upload_request: UploadDatasetRequest) -> UploadDatasetCreateResponse:
 51        """
 52        Registers a dataset in Cirro, which can subsequently have files uploaded to it
 53
 54        Args:
 55            project_id (str): ID of the Project
 56            upload_request (cirro_api_client.v1.models.UploadDatasetRequest):
 57
 58        Returns:
 59            ID of the created dataset and the path to upload files
 60
 61        ```python
 62        from cirro_api_client.v1.models import UploadDatasetRequest
 63        from cirro.cirro_client import CirroApi
 64
 65        cirro = CirroApi()
 66        request = UploadDatasetRequest(
 67            name="Name of new dataset",
 68            process_id="paired_dnaseq",
 69            expected_files=["read_1.fastq.gz", "read_2.fastq.gz"],
 70            description="Description of the dataset"
 71        )
 72        cirro.datasets.create("project-id", request)
 73        ```
 74        """
 75        return upload_dataset.sync(project_id=project_id, client=self._api_client, body=upload_request)
 76
 77    def get(self, project_id: str, dataset_id: str) -> Optional[DatasetDetail]:
 78        """
 79        Gets detailed information about a dataset
 80
 81        Args:
 82            project_id (str): ID of the Project
 83            dataset_id (str): ID of the Dataset
 84
 85        Returns:
 86            The dataset, if found
 87        """
 88        return get_dataset.sync(project_id=project_id, dataset_id=dataset_id, client=self._api_client)
 89
 90    def update(self, project_id: str, dataset_id: str, request: UpdateDatasetRequest) -> DatasetDetail:
 91        """
 92        Update info on a dataset (name, description, and/or tags)
 93
 94        Args:
 95            project_id (str): ID of the Project
 96            dataset_id (str): ID of the Dataset
 97            request (cirro_api_client.v1.models.UpdateDatasetRequest):
 98
 99        Returns:
100            The updated dataset
101
102        ```python
103        from cirro_api_client.v1.models import UpdateDatasetRequest
104        from cirro.cirro_client import CirroApi
105
106        cirro = CirroApi()
107        request = UpdateDatasetRequest(
108            name="Name of new dataset",
109            process_id="paired_dnaseq",
110            description="Description of the dataset"
111        )
112        cirro.datasets.update("project-id", "dataset-id", request)
113        ```
114        """
115        return update_dataset.sync(project_id=project_id, dataset_id=dataset_id, body=request, client=self._api_client)
116
117    def delete(self, project_id: str, dataset_id: str) -> None:
118        """
119        Delete a dataset
120
121        After a dataset has been deleted, the files associated with that
122        dataset are saved according to the project's retention time.
123
124        Args:
125            project_id (str): ID of the Project
126            dataset_id (str): ID of the Dataset
127        """
128        delete_dataset.sync_detailed(project_id=project_id, dataset_id=dataset_id, client=self._api_client)
129
130    def get_file_listing(self, project_id: str, dataset_id: str) -> List[File]:
131        """
132        Gets a listing of files, charts, and other assets available for the dataset
133
134        Args:
135            project_id (str): ID of the Project
136            dataset_id (str): ID of the Dataset
137        """
138        manifest = get_dataset_manifest.sync(
139            project_id=project_id,
140            dataset_id=dataset_id,
141            client=self._api_client
142        )
143
144        files = [
145            File.from_file_entry(
146                f,
147                project_id=project_id,
148                domain=manifest.domain
149            )
150            for f in manifest.files
151        ]
152        return files
153
154    def upload_files(self, project_id: str, dataset_id: str, local_directory: str, files: List[str]) -> None:
155        """
156        Uploads files to a given dataset from the specified local directory
157
158        Args:
159            project_id (str): ID of the Project
160            dataset_id (str): ID of the Dataset
161            local_directory (str): Path to local directory
162            files (typing.List[str]): List of relative paths to files within the local directory
163
164        """
165        dataset = self.get(project_id, dataset_id)
166
167        access_context = FileAccessContext.upload_dataset(
168            project_id=project_id,
169            dataset_id=dataset_id,
170            base_url=dataset.s3
171        )
172
173        self._file_service.upload_files(
174            access_context,
175            local_directory,
176            files
177        )
178
179    def download_files(
180        self,
181        project_id: str,
182        dataset_id: str,
183        download_location: str,
184        files: Union[List[File], List[str]] = None
185    ) -> None:
186        """
187        Downloads files from a dataset
188
189        The `files` argument is used to optionally specify a subset of files
190        to be downloaded. By default, all files are downloaded.
191
192        Args:
193            project_id (str): ID of the Project
194            dataset_id (str): ID of the Dataset
195            download_location (str): Local destination for downloaded files
196            files (typing.List[str]): Optional list of files to download
197        """
198        if files is None:
199            files = self.get_file_listing(project_id, dataset_id)
200
201        if len(files) == 0:
202            return
203
204        first_file = files[0]
205        if isinstance(first_file, File):
206            files = [file.relative_path for file in files]
207            access_context = first_file.access_context
208        else:
209            dataset = self.get(project_id, dataset_id)
210            access_context = FileAccessContext.download(project_id=project_id,
211                                                        base_url=dataset.s3)
212
213        self._file_service.download_files(access_context, download_location, files)

 13class ExecutionService(BaseService):
 14    """
 15    Service for interacting with the Execution endpoints
 16    """
 17    def run_analysis(self, project_id: str, request: RunAnalysisRequest) -> CreateResponse:
 18        """
 19        Launch an analysis job running a process on a set of inputs
 20
 21        Args:
 22            project_id (str): ID of the Project
 23            request (cirro_api_client.v1.models.RunAnalysisRequest):
 24
 25        Returns:
 26            The ID of the created dataset
 27
 28        ```python
 29        from cirro_api_client.v1.models import RunAnalysisRequest, RunAnalysisRequestParams
 30        from cirro.cirro_client import CirroApi
 31
 32        # Example:
 33        # Run the "process-nf-core-rnaseq-3_8" process using input data
 34        # from a dataset with the id "source-dataset-id"
 35
 36        # Optional analysis parameters
 37        params = RunAnalysisRequestParams.from_dict({
 38            "param_a": "val_a",
 39            "param_b": "val_b"
 40        })
 41
 42        cirro = CirroApi()
 43        request = RunAnalysisRequest(
 44            name="Name of the newly created dataset",
 45            description="Longer description of newly created dataset",
 46            process_id="process-nf-core-rnaseq-3_8",
 47            source_dataset_ids=["source-dataset-id"],
 48            params=params
 49        )
 50        cirro.execution.run_analysis("project-id", request)
 51        ```
 52        """
 53
 54        form_spec = get_process_parameters.sync(
 55            process_id=request.process_id,
 56            client=self._api_client
 57        )
 58
 59        ParameterSpecification(
 60            form_spec
 61        ).validate_params(
 62            request.params.to_dict() if request.params else {}
 63        )
 64
 65        return run_analysis.sync(
 66            project_id=project_id,
 67            body=request,
 68            client=self._api_client
 69        )
 70
 71    def stop_analysis(self, project_id: str, dataset_id: str):
 72        """
 73        Terminates all jobs related to a running analysis
 74
 75        Args:
 76            project_id (str): ID of the Project
 77            dataset_id (str): ID of the Dataset
 78        """
 79
 80        return stop_analysis.sync(
 81            project_id=project_id,
 82            dataset_id=dataset_id,
 83            client=self._api_client
 84        )
 85
 86    def get_project_summary(self, project_id: str) -> GetProjectSummaryResponse200:
 87        """
 88        Gets an overview of the executions currently running in the project, by job queue
 89
 90        Args:
 91            project_id (str): ID of the Project
 92
 93        Returns:
 94            `cirro_api_client.v1.models.GetProjectSummaryResponse200`
 95        """
 96
 97        return get_project_summary.sync(
 98            project_id=project_id,
 99            client=self._api_client
100        ).additional_properties
101
102    def get_execution_logs(self, project_id: str, dataset_id: str) -> str:
103        """
104        Gets live logs from main execution task
105
106        Args:
107            project_id (str): ID of the Project
108            dataset_id (str): ID of the Dataset
109
110        """
111
112        resp = get_execution_logs.sync(
113            project_id=project_id,
114            dataset_id=dataset_id,
115            client=self._api_client
116        )
117
118        return '\n'.join(e.message for e in resp.events)
119
120    def get_tasks_for_execution(self, project_id: str, dataset_id: str) -> Optional[List[Task]]:
121        """
122        Gets the tasks submitted by the workflow execution
123
124        Args:
125            project_id (str): ID of the Project
126            dataset_id (str): ID of the Dataset
127        """
128
129        return get_tasks_for_execution.sync(
130            project_id=project_id,
131            dataset_id=dataset_id,
132            client=self._api_client
133        )
134
135    def get_task_logs(self, project_id: str, dataset_id: str, task_id: str) -> str:
136        """
137        Gets the log output from an individual task
138
139        Args:
140            project_id (str): ID of the Project
141            dataset_id (str): ID of the Dataset
142            task_id (str): ID of the task
143
144        """
145
146        resp = get_task_logs.sync(
147            project_id=project_id,
148            dataset_id=dataset_id,
149            task_id=task_id,
150            client=self._api_client
151        )
152
153        return '\n'.join(e.message for e in resp.events)

 17class FileService(BaseService):
 18    """
 19    Service for interacting with files
 20    """
 21    enable_additional_checksum: bool
 22    transfer_retries: int
 23    _get_token_lock = threading.Lock()
 24    _read_token_cache: Dict[str, AWSCredentials] = {}
 25
 26    def __init__(self, api_client, enable_additional_checksum, transfer_retries):
 27        """
 28        Instantiates the file service class
 29        """
 30        self._api_client = api_client
 31        self.enable_additional_checksum = enable_additional_checksum
 32        self.transfer_retries = transfer_retries
 33
 34    def get_access_credentials(self, access_context: FileAccessContext) -> AWSCredentials:
 35        """
 36        Retrieves credentials to access files
 37
 38        Args:
 39            access_context (cirro.models.file.FileAccessContext): File access context, use class methods to generate
 40        """
 41        access_request = access_context.file_access_request
 42
 43        if access_request.access_type == AccessType.PROJECT_DOWNLOAD:
 44            return self._get_project_read_credentials(access_context)
 45
 46        else:
 47            return generate_project_file_access_token.sync(
 48                client=self._api_client,
 49                project_id=access_context.project_id,
 50                body=access_context.file_access_request
 51            )
 52
 53    def _get_project_read_credentials(self, access_context: FileAccessContext):
 54        """
 55        Retrieves credentials to read project data, this can be cached
 56        """
 57        access_request = access_context.file_access_request
 58        project_id = access_context.project_id
 59        with self._get_token_lock:
 60            cached_token = self._read_token_cache.get(project_id)
 61
 62            if not cached_token or datetime.now(tz=timezone.utc) > cached_token.expiration:
 63                new_token = generate_project_file_access_token.sync(
 64                    client=self._api_client,
 65                    project_id=project_id,
 66                    body=access_request
 67                )
 68
 69                self._read_token_cache[project_id] = new_token
 70
 71        return self._read_token_cache[project_id]
 72
 73    def get_file(self, file: File) -> bytes:
 74        """
 75        Gets the contents of a file
 76
 77        Args:
 78            file (cirro.models.file.File):
 79
 80        Returns:
 81            The raw bytes of the file
 82        """
 83        return self.get_file_from_path(file.access_context, file.relative_path)
 84
 85    def get_file_from_path(self, access_context: FileAccessContext, file_path: str) -> bytes:
 86        """
 87        Gets the contents of a file by providing the path, used internally
 88
 89        Args:
 90            access_context (cirro.models.file.FileAccessContext): File access context, use class methods to generate
 91            file_path (str): Relative path to file within dataset
 92
 93        Returns:
 94            The raw bytes of the file
 95        """
 96
 97        s3_client = S3Client(
 98            partial(self.get_access_credentials, access_context),
 99            self.enable_additional_checksum
100        )
101
102        full_path = f'{access_context.prefix}/{file_path}'.lstrip('/')
103
104        return s3_client.get_file(access_context.bucket, full_path)
105
106    def create_file(self, access_context: FileAccessContext, key: str,
107                    contents: str, content_type: str) -> None:
108        """
109        Creates a file at the specified path
110
111        Args:
112            access_context (cirro.models.file.FileAccessContext): File access context, use class methods to generate
113            key (str): Key of object to create
114            contents (str): Content of object
115            content_type (str):
116        """
117
118        s3_client = S3Client(
119            partial(self.get_access_credentials, access_context),
120            self.enable_additional_checksum
121        )
122
123        s3_client.create_object(
124            key=key,
125            contents=contents,
126            content_type=content_type,
127            bucket=access_context.bucket
128        )
129
130    def upload_files(self, access_context: FileAccessContext, directory: str, files: List[str]) -> None:
131        """
132        Uploads a list of files from the specified directory
133
134        Args:
135            access_context (cirro.models.file.FileAccessContext): File access context, use class methods to generate
136            directory (str): base path to upload from
137            files (List[str]): relative path of files to upload
138        """
139
140        s3_client = S3Client(
141            partial(self.get_access_credentials, access_context),
142            self.enable_additional_checksum
143        )
144
145        upload_directory(
146            directory,
147            files,
148            s3_client,
149            access_context.bucket,
150            access_context.prefix,
151            max_retries=self.transfer_retries
152        )
153
154    def download_files(self, access_context: FileAccessContext, directory: str, files: List[str]) -> None:
155        """
156        Download a list of files to the specified directory
157
158        Args:
159            access_context (cirro.models.file.FileAccessContext): File access context, use class methods to generate
160            directory (str): download location
161            files (List[str]): relative path of files to download
162        """
163        s3_client = S3Client(
164            partial(self.get_access_credentials, access_context),
165            self.enable_additional_checksum
166        )
167
168        download_directory(
169            directory,
170            files,
171            s3_client,
172            access_context.bucket,
173            access_context.prefix
174        )

11class MetadataService(BaseService):
12    """
13    Service for interacting with the Metadata endpoints
14    """
15    def get_project_samples(self, project_id: str, max_items: int = 10000) -> List[Sample]:
16        """
17        Retrieves a list of samples associated with a project along with their metadata
18
19        Args:
20            project_id (str): ID of the Project
21            max_items (int): Maximum number of records to get (default 10,000)
22        """
23        return get_all_records(
24            records_getter=lambda page_args: get_project_samples.sync(project_id=project_id,
25                                                                      client=self._api_client,
26                                                                      next_token=page_args.next_token,
27                                                                      limit=page_args.limit),
28            max_items=max_items
29        )
30
31    def get_project_schema(self, project_id: str) -> FormSchema:
32        """
33        Get project metadata schema
34
35        Args:
36            project_id (str): ID of the Project
37        """
38        return get_project_schema.sync(project_id=project_id, client=self._api_client)
39
40    def update_project_schema(self, project_id: str, schema: FormSchema):
41        """
42        Update project metadata schema
43
44        Args:
45            project_id (str): ID of the Project
46            schema (cirro_api_client.v1.models.FormSchema): Metadata schema
47        """
48        update_project_schema.sync_detailed(project_id=project_id, body=schema, client=self._api_client)
49
50    def update_sample(self, project_id: str, sample_id: str, sample: SampleRequest) -> Sample:
51        """
52        Updates metadata information for sample
53
54        Args:
55            project_id (str): ID of the Project
56            sample_id (str): ID of the sample
57            sample (cirro_api_client.v1.models.SampleRequest): Metadata information for the sample
58        """
59        return update_sample.sync(
60            project_id=project_id,
61            sample_id=sample_id,
62            body=sample,
63            client=self._api_client
64        )