cirro

 13class DataPortal:
 14    """
 15    Helper functions for exploring the Projects, Datasets, Samples, and Files
 16    available in the Data Portal.
 17    """
 18
 19    def __init__(self, base_url: str = None, client: CirroApi = None):
 20        """
 21        Set up the DataPortal object, establishing an authenticated connection.
 22
 23        Args:
 24            base_url (str): Optional base URL of the Cirro instance
 25             (if not provided, it uses the `CIRRO_BASE_URL` environment variable, or the config file)
 26            client (`cirro.cirro_client.CirroApi`): Optional pre-configured client
 27
 28        Example:
 29        ```python
 30        from cirro import DataPortal
 31
 32        portal = DataPortal(base_url="app.cirro.bio")
 33        portal.list_projects()
 34        ```
 35        """
 36
 37        if client is not None:
 38            self._client = client
 39
 40        # Set up default client if not provided
 41        else:
 42            self._client = CirroApi(base_url=base_url)
 43
 44    def list_projects(self) -> DataPortalProjects:
 45        """List all the projects available in the Data Portal."""
 46
 47        return DataPortalProjects(
 48            [
 49                DataPortalProject(proj, self._client)
 50                for proj in self._client.projects.list()
 51            ]
 52        )
 53
 54    def get_project_by_name(self, name: str = None) -> DataPortalProject:
 55        """Return the project with the specified name."""
 56
 57        return self.list_projects().get_by_name(name)
 58
 59    def get_project_by_id(self, _id: str = None) -> DataPortalProject:
 60        """Return the project with the specified id."""
 61
 62        return self.list_projects().get_by_id(_id)
 63
 64    def get_project(self, project: str = None) -> DataPortalProject:
 65        """
 66        Return a project identified by ID or name.
 67
 68        Args:
 69            project (str): ID or name of project
 70
 71        Returns:
 72            `from cirro.sdk.project import DataPortalProject`
 73        """
 74        try:
 75            return self.get_project_by_id(project)
 76        except DataPortalAssetNotFound:
 77            return self.get_project_by_name(project)
 78
 79    def get_dataset(self, project: str = None, dataset: str = None) -> DataPortalDataset:
 80        """
 81        Return a dataset identified by ID or name.
 82
 83        Args:
 84            project (str): ID or name of project
 85            dataset (str): ID or name of dataset
 86
 87        Returns:
 88            `cirro.sdk.dataset.DataPortalDataset`
 89
 90            ```python
 91            from cirro import DataPortal()
 92            portal = DataPortal()
 93            dataset = portal.get_dataset(
 94                project="id-or-name-of-project",
 95                dataset="id-or-name-of-dataset"
 96            )
 97            ```
 98        """
 99        try:
100            project: DataPortalProject = self.get_project_by_id(project)
101        except DataPortalAssetNotFound:
102            project: DataPortalProject = self.get_project_by_name(project)
103
104        return project.get_dataset(dataset)
105
106    def read_files(
107            self,
108            project: str,
109            dataset: str,
110            glob: str = None,
111            pattern: str = None,
112            filetype: str = None,
113            **kwargs
114    ):
115        """
116        Read the contents of files from a dataset.
117
118        The project and dataset can each be identified by name or ID.
119        Exactly one of ``glob`` or ``pattern`` must be provided.
120
121        **glob** — standard wildcard matching; yields the file content for each
122        matching file:
123
124        - ``*`` matches any characters within a single path segment
125        - ``**`` matches zero or more path segments
126        - Matching is suffix-anchored (``*.csv`` matches at any depth)
127
128        **pattern** — like ``glob`` but ``{name}`` placeholders capture portions
129        of the path automatically; yields ``(content, meta)`` pairs where
130        *meta* is a ``dict`` of extracted values:
131
132        - ``{name}`` captures one path segment (no ``/``)
133        - ``*`` and ``**`` wildcards work as in ``glob``
134
135        Args:
136            project (str): ID or name of the project.
137            dataset (str): ID or name of the dataset.
138            glob (str): Wildcard expression to match files
139                (e.g., ``'*.csv'``, ``'data/**/*.tsv.gz'``).
140                Yields one item per matching file: the parsed content.
141            pattern (str): Wildcard expression with ``{name}`` capture
142                placeholders (e.g., ``'{sample}.csv'``,
143                ``'{condition}/{sample}.csv'``).
144                Yields ``(content, meta)`` per matching file.
145            filetype (str): File format used to parse each file. Supported values:
146
147                - ``'csv'``: parse with :func:`pandas.read_csv`, returns a ``DataFrame``
148                - ``'h5ad'``: parse as AnnData (requires ``anndata`` package)
149                - ``'json'``: parse with :func:`json.loads`, returns a Python object
150                - ``'parquet'``: parse with :func:`pandas.read_parquet`, returns a ``DataFrame``
151                  (requires ``pyarrow`` or ``fastparquet``)
152                - ``'feather'``: parse with :func:`pandas.read_feather`, returns a ``DataFrame``
153                  (requires ``pyarrow``)
154                - ``'pickle'``: deserialize with :mod:`pickle`, returns a Python object
155                - ``'excel'``: parse with :func:`pandas.read_excel`, returns a ``DataFrame``
156                  (requires ``openpyxl`` for ``.xlsx`` or ``xlrd`` for ``.xls``)
157                - ``'text'``: read as plain text, returns a ``str``
158                - ``'bytes'``: read as raw bytes, returns ``bytes``
159                - ``None`` (default): infer from file extension
160                  (``.csv``/``.tsv`` → ``'csv'``, ``.h5ad`` → ``'h5ad'``,
161                  ``.json`` → ``'json'``, ``.parquet`` → ``'parquet'``,
162                  ``.feather`` → ``'feather'``, ``.pkl``/``.pickle`` → ``'pickle'``,
163                  ``.xlsx``/``.xls`` → ``'excel'``, otherwise ``'text'``)
164            **kwargs: Additional keyword arguments forwarded to the file-parsing
165                function (e.g., ``sep='\\t'`` for CSV/TSV files).
166
167        Yields:
168            - When using ``glob``: *content* for each matching file
169            - When using ``pattern``: ``(content, meta)`` for each matching file,
170              where *meta* is a ``dict`` of values extracted from ``{name}``
171              placeholders
172
173        Raises:
174            DataPortalInputError: if both ``glob`` and ``pattern`` are provided,
175                or if neither is provided.
176
177        Example:
178            ```python
179            # Read all CSV files — just the content
180            for df in portal.read_files('My Project', 'My Dataset', glob='*.csv'):
181                print(df.shape)
182
183            # Extract sample names from filenames automatically
184            for df, meta in portal.read_files('My Project', 'My Dataset', pattern='{sample}.csv'):
185                print(meta['sample'], df.shape)
186
187            # Multi-level capture: condition directory + sample filename
188            for df, meta in portal.read_files('My Project', 'My Dataset', pattern='{condition}/{sample}.csv'):
189                print(meta['condition'], meta['sample'], df.shape)
190
191            # Read gzip-compressed TSV files with explicit separator
192            for df in portal.read_files('My Project', 'My Dataset', glob='**/*.tsv.gz', filetype='csv', sep='\\t'):
193                print(df.shape)
194            ```
195        """
196        ds = self.get_dataset(project=project, dataset=dataset)
197        yield from ds.read_files(glob=glob, pattern=pattern, filetype=filetype, **kwargs)
198
199    def read_file(
200            self,
201            project: str,
202            dataset: str,
203            path: str = None,
204            glob: str = None,
205            filetype: str = None,
206            **kwargs
207    ):
208        """
209        Read the contents of a single file from a dataset.
210
211        The project and dataset can each be identified by name or ID.
212        Provide either ``path`` (exact relative path) or ``glob`` (wildcard
213        expression). If ``glob`` is used it must match exactly one file.
214
215        Args:
216            project (str): ID or name of the project.
217            dataset (str): ID or name of the dataset.
218            path (str): Exact relative path of the file within the dataset.
219            glob (str): Wildcard expression matching exactly one file.
220            filetype (str): File format used to parse the file. Supported values
221                are the same as :meth:`read_files`.
222            **kwargs: Additional keyword arguments forwarded to the
223                file-parsing function.
224
225        Returns:
226            Parsed file content.
227
228        Raises:
229            DataPortalInputError: if both or neither of ``path``/``glob`` are
230                provided, or if ``glob`` matches zero or more than one file.
231        """
232        ds = self.get_dataset(project=project, dataset=dataset)
233        return ds.read_file(path=path, glob=glob, filetype=filetype, **kwargs)
234
235    def list_processes(self, ingest=False) -> DataPortalProcesses:
236        """
237        List all the processes available in the Data Portal.
238        By default, only list non-ingest processes (those which can be run on existing datasets).
239        To list the processes which can be used to upload datasets, use `ingest = True`.
240
241        Args:
242            ingest (bool): If True, only list those processes which can be used to ingest datasets directly
243        """
244
245        return DataPortalProcesses(
246            [
247                DataPortalProcess(p, self._client)
248                for p in self._client.processes.list()
249                if not ingest or p.executor == Executor.INGEST
250            ]
251        )
252
253    def get_process_by_name(self, name: str, ingest=False) -> DataPortalProcess:
254        """
255        Return the process with the specified name.
256
257        Args:
258            name (str): Name of process
259        """
260
261        return self.list_processes(ingest=ingest).get_by_name(name)
262
263    def get_process_by_id(self, id: str, ingest=False) -> DataPortalProcess:
264        """
265        Return the process with the specified id
266
267        Args:
268            id (str): ID of process
269        """
270
271        return self.list_processes(ingest=ingest).get_by_id(id)
272
273    def list_reference_types(self) -> DataPortalReferenceTypes:
274        """
275        Return the list of all available reference types
276        """
277
278        return DataPortalReferenceTypes(
279            [
280                DataPortalReferenceType(ref)
281                for ref in self._client.references.get_types()
282            ]
283        )
284
285    @property
286    def developer_helper(self) -> DeveloperHelper:
287        return DeveloperHelper(self._client)

 8class DataPortalLogin:
 9    """
10    Start the login process, obtaining the authorization message from Cirro
11    needed to confirm the user identity.
12
13    Useful when you need to authenticate a user in a non-blocking way.
14
15    Usage:
16
17    ```python
18    # Replace app.cirro.bio as appropriate
19    login = DataPortalLogin(base_url="app.cirro.bio")
20
21    # Present the user with the authorization message
22    print(login.auth_message)
23
24    # Generate the authenticated DataPortal object,
25    # blocking until the user completes the login process in their browser
26    portal = login.await_completion()
27    ```
28    """
29    base_url: str
30    auth_info: DeviceCodeAuth
31
32    def __init__(self, base_url: str = None, enable_cache=False):
33        app_config = AppConfig(base_url=base_url)
34
35        self.base_url = base_url
36
37        self.auth_info = DeviceCodeAuth(
38            region=app_config.region,
39            client_id=app_config.client_id,
40            auth_endpoint=app_config.auth_endpoint,
41            enable_cache=enable_cache,
42            await_completion=False
43        )
44
45    @property
46    def auth_message(self) -> str:
47        """Authorization message provided by Cirro."""
48        return self.auth_info.auth_message
49
50    @property
51    def auth_message_markdown(self) -> str:
52        """Authorization message provided by Cirro (Markdown format)."""
53        return self.auth_info.auth_message_markdown
54
55    def await_completion(self) -> DataPortal:
56        """Complete the login process and return an authenticated client"""
57
58        # Block until the user completes the login flow
59        self.auth_info.await_completion()
60
61        # Set up the client object
62        cirro_client = CirroApi(
63            auth_info=self.auth_info,
64            base_url=self.base_url
65        )
66
67        # Return the Data Portal object
68        return DataPortal(client=cirro_client)

 20class DataPortalProject(DataPortalAsset):
 21    """
 22    Projects in the Data Portal contain collections of Datasets.
 23    Users are granted permissions at the project-level, allowing them
 24    to view and/or modify all the datasets in that collection.
 25    """
 26    def __init__(self, proj: Project, client: CirroApi):
 27        """
 28        Instantiate with helper method
 29
 30        ```python
 31        from cirro import DataPortal()
 32        portal = DataPortal()
 33        project = portal.get_project_by_name("Project Name")
 34        ```
 35
 36        """
 37        self._data = proj
 38        self._client = client
 39
 40    @property
 41    def id(self) -> str:
 42        """
 43        Unique identifier
 44        """
 45        return self._data.id
 46
 47    @property
 48    def name(self) -> str:
 49        """
 50        Readable name
 51        """
 52        return self._data.name
 53
 54    @property
 55    def description(self) -> str:
 56        """
 57        Longer description of the project
 58        """
 59        return self._data.description
 60
 61    @property
 62    def status(self) -> Status:
 63        """
 64        Status of the project
 65        """
 66        return self._data.status
 67
 68    def __str__(self):
 69        """Control how the Project is rendered as a string."""
 70
 71        return '\n'.join([
 72            f"{i.title()}: {self.__getattribute__(i)}"
 73            for i in ['name', 'id', 'description']
 74        ])
 75
 76    @cache
 77    def _get_datasets(self) -> List[Dataset]:
 78        return list_all_datasets(project_id=self.id,
 79                                 client=self._client)
 80
 81    def list_datasets(self, force_refresh=False) -> DataPortalDatasets:
 82        """List all the datasets available in the project."""
 83        if force_refresh:
 84            self._get_datasets.cache_clear()
 85
 86        return DataPortalDatasets(
 87            [
 88                DataPortalDataset(d, self._client)
 89                for d in self._get_datasets()
 90            ]
 91        )
 92
 93    def get_dataset(self, name_or_id: str, force_refresh=False) -> DataPortalDataset:
 94        """Return the dataset matching the given ID or name.
 95
 96        Tries to match by ID first, then by name.
 97        Raises an error if the name matches multiple datasets.
 98        """
 99        if force_refresh:
100            self._get_datasets.cache_clear()
101
102        # Try by ID first
103        try:
104            return self.get_dataset_by_id(name_or_id)
105        except Exception:
106            pass
107
108        # Fall back to name matching
109        matches = [d for d in self._get_datasets() if d.name == name_or_id]
110        if len(matches) == 0:
111            raise DataPortalAssetNotFound(f'Dataset with name or ID "{name_or_id}" not found')
112        if len(matches) > 1:
113            raise DataPortalInputError(
114                f'Multiple datasets found with the name "{name_or_id}" — use get_dataset_by_id instead'
115            )
116        return self.get_dataset_by_id(matches[0].id)
117
118    def get_dataset_by_name(self, name: str, force_refresh=False) -> DataPortalDataset:
119        """Return the dataset with the specified name."""
120        if force_refresh:
121            self._get_datasets.cache_clear()
122
123        dataset = next((d for d in self._get_datasets() if d.name == name), None)
124        if dataset is None:
125            raise DataPortalAssetNotFound(f'Dataset with name {name} not found')
126        return self.get_dataset_by_id(dataset.id)
127
128    def get_dataset_by_id(self, _id: str = None) -> DataPortalDataset:
129        """Return the dataset with the specified id."""
130
131        dataset = self._client.datasets.get(project_id=self.id, dataset_id=_id)
132        if dataset is None:
133            raise DataPortalAssetNotFound(f'Dataset with ID {_id} not found')
134        return DataPortalDataset(dataset, self._client)
135
136    def list_references(self, reference_type: str = None) -> DataPortalReferences:
137        """
138        List the references available in a project.
139        Optionally filter to references of a particular type (identified by name)
140        """
141
142        # Get the complete list of references which are available
143        reference_types = DataPortalReferenceTypes(
144            [
145                DataPortalReferenceType(ref)
146                for ref in self._client.references.get_types()
147            ]
148        )
149
150        # If a particular name was specified
151        if reference_type is not None:
152            reference_types = reference_types.filter_by_pattern(reference_type)
153            if len(reference_types) == 0:
154                msg = f"Could not find any reference types with the name {reference_type}"
155                raise DataPortalAssetNotFound(msg)
156
157        return DataPortalReferences(
158            [
159                DataPortalReference(ref, project_id=self.id, client=self._client)
160                for ref in self._client.references.get_for_project(
161                    self.id
162                )
163                if reference_type is None or ref.type_ == reference_type
164            ]
165        )
166
167    def get_reference_by_name(self, name: str = None, ref_type: str = None) -> DataPortalReference:
168        """Return the reference of a particular type with the specified name."""
169
170        if name is None:
171            raise DataPortalInputError("Must specify the reference name")
172
173        return self.list_references(ref_type).get_by_name(name)
174
175    def upload_dataset(
176        self,
177        name: str = None,
178        description='',
179        process: Union[DataPortalProcess, str] = None,
180        upload_folder: str = None,
181        files: List[str] = None,
182        tags: List[str] = None,
183    ):
184        """
185        Upload a set of files to the Data Portal, creating a new dataset.
186
187        If the files parameter is not provided, it will upload all files in the upload folder
188
189        Args:
190            name (str): Name of newly created dataset
191            description (str): Description of newly created dataset
192            process (str | DataPortalProcess): Process to run may be referenced by name, ID, or object
193            upload_folder (str): Folder containing files to upload
194            files (List[str]): Optional subset of files to upload from the folder
195            tags (List[str]): Optional list of tags to apply to the dataset
196        """
197
198        if name is None:
199            raise DataPortalInputError("Must provide name for new dataset")
200        if process is None:
201            raise DataPortalInputError("Must provide the process which is used for ingest")
202        if upload_folder is None:
203            raise DataPortalInputError("Must provide upload_folder -- folder containing files to upload")
204
205        # Parse the process provided by the user
206        process = parse_process_name_or_id(process, self._client)
207
208        # If no files were provided
209        if files is None:
210            # Get the list of files in the upload folder
211            files = get_files_in_directory(upload_folder)
212
213        if files is None or len(files) == 0:
214            raise RuntimeWarning("No files to upload, exiting")
215
216        # Normalize into Tag object
217        if tags is not None:
218            tags = [Tag(value=value) for value in tags]
219
220        # Make sure that the files match the expected pattern
221        self._client.processes.check_dataset_files(files, process.id, upload_folder)
222
223        # Create the ingest process request
224        dataset_create_request = UploadDatasetRequest(
225            process_id=process.id,
226            name=name,
227            description=description,
228            expected_files=files,
229            tags=tags,
230        )
231
232        # Get the response
233        create_response = self._client.datasets.create(project_id=self.id,
234                                                       upload_request=dataset_create_request)
235
236        # Upload the files
237        self._client.datasets.upload_files(
238            project_id=self.id,
239            dataset_id=create_response.id,
240            directory=upload_folder,
241            files=files
242        )
243
244        # Return the dataset which was created, which might take a second to update
245        max_attempts = 5
246        for attempt in range(max_attempts):
247            try:
248                return self.get_dataset_by_id(create_response.id)
249            except DataPortalAssetNotFound as e:
250                if attempt == max_attempts - 1:
251                    raise e
252                else:
253                    sleep(2)
254
255    def samples(self, max_items: int = 10000) -> List[Sample]:
256        """
257        Retrieves a list of samples associated with a project along with their metadata
258
259        Args:
260            max_items (int): Maximum number of records to get (default 10,000)
261        """
262        return self._client.metadata.get_project_samples(self.id, max_items)