diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..d0976cb --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,40 @@ +name: Publish to PyPI + +on: + push: + tags: + - 'v*' # Trigger on version tags like v1.0.0 + release: + types: [published] # Also trigger on GitHub releases + workflow_dispatch: # Allow manual triggering + +jobs: + build-and-publish: + runs-on: ubuntu-latest + + environment: + name: pypi + url: https://pypi.org/project/feedly-client/ + + permissions: + id-token: write # IMPORTANT: OIDC token generation + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + + - name: Build package + run: python -m build + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + diff --git a/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_csv.py b/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_csv.py new file mode 100644 index 0000000..f3afe75 --- /dev/null +++ b/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_csv.py @@ -0,0 +1,46 @@ +import json +from csv import DictWriter +from datetime import datetime, timedelta +from pathlib import Path +from pprint import pprint + +from feedly.api_client.enterprise.indicators_of_compromise import IoCDownloaderBuilder, IoCFormat +from feedly.api_client.session import FeedlySession +from feedly.api_client.utils import run_example + +RESULTS_DIR = Path(__file__).parent / "results" +RESULTS_DIR.mkdir(exist_ok=True) + + +def example_export_indicators_of_compromise_from_all_enterprise_feeds_as_csv(): + """ + This example will save a CSV file containing the contextualized IoCs that Leo extracted during the past 12 + hours in all your enterprise feeds. + """ + # Authenticate using the default auth directory + session = FeedlySession() + + # Create the CSV IoC downloader builder object, and limit it to 12 hours + # Usually newer_than will be the datetime of the last fetch + downloader_builder = IoCDownloaderBuilder( + session=session, newer_than=datetime.now() - timedelta(hours=12), format=IoCFormat.CSV + ) + + # Fetch the IoC from all the enterprise categories + # You can use a different method to get the iocs from you personal categories, personal or enterprise boards, + # or from specific categories/boards using their names or ids + downloader = downloader_builder.from_all_enterprise_categories() + iocs = downloader.download_all() + + # Save the IoCs in a CSV + with (RESULTS_DIR / "ioc_example.csv").open("w") as f: + writer = DictWriter(f, fieldnames=list(iocs[0].keys())) + writer.writerows(iocs) + + +if __name__ == "__main__": + # Warning: This example requires the Threat Intelligence package to be enabled on your account + + # Will prompt for the token if missing, and launch the example above + # If a token expired error is raised, will prompt for a new token and restart the example + run_example(example_export_indicators_of_compromise_from_all_enterprise_feeds_as_csv) diff --git a/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_stix.py b/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_stix.py index d93bc11..74b5d07 100644 --- a/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_stix.py +++ b/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_stix.py @@ -40,6 +40,8 @@ def example_export_indicators_of_compromise_from_all_enterprise_feeds_as_stix(): if __name__ == "__main__": + # Warning: This example requires the Threat Intelligence package to be enabled on your account + # Will prompt for the token if missing, and launch the example above # If a token expired error is raised, will prompt for a new token and restart the example run_example(example_export_indicators_of_compromise_from_all_enterprise_feeds_as_stix) diff --git a/examples/enterprise/export_indicators_of_compromise_to_misp.py b/examples/enterprise/export_indicators_of_compromise_to_misp.py index e385e07..6594df5 100644 --- a/examples/enterprise/export_indicators_of_compromise_to_misp.py +++ b/examples/enterprise/export_indicators_of_compromise_to_misp.py @@ -38,6 +38,8 @@ def export_indicators_of_compromise_to_misp(): if __name__ == "__main__": + # Warning: This example requires the Threat Intelligence package to be enabled on your account + logging.basicConfig(level="INFO") filterwarnings("ignore") diff --git a/feedly/api_client/enterprise/indicators_of_compromise.py b/feedly/api_client/enterprise/indicators_of_compromise.py index 94b3b49..8d2484d 100644 --- a/feedly/api_client/enterprise/indicators_of_compromise.py +++ b/feedly/api_client/enterprise/indicators_of_compromise.py @@ -1,13 +1,13 @@ import uuid from abc import ABC, abstractmethod +from csv import DictReader from datetime import datetime from enum import Enum from itertools import chain +from requests import Response from typing import ClassVar, Dict, Generic, Iterable, List, Optional, TypeVar from urllib.parse import parse_qs -from requests import Response - from feedly.api_client.data import Streamable from feedly.api_client.session import FeedlySession @@ -17,44 +17,68 @@ class IoCFormat(Enum): MISP = "misp" STIX = "stix2.1" + CSV = "csv" class IoCDownloaderABC(ABC, Generic[T]): RELATIVE_URL = "/v3/enterprise/ioc" FORMAT: ClassVar[str] - def __init__(self, session: FeedlySession, newer_than: Optional[datetime], stream_id: str): + def __init__( + self, + session: FeedlySession, + newer_than: Optional[datetime], + older_than: Optional[datetime], + stream_id: str, + max_batches: Optional[int] = None, + ): """ Use this class to export the contextualized IoCs from a stream. Enterprise/personals feeds/boards are supported (see dedicated methods below). The IoCs will be returned along with their context and relationships in a dictionary representing a valid STIX v2.1 Bundle object. https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_gms872kuzdmg Use the newer_than parameter to filter articles that are newer than your last call. + Use the older_than parameter to filter articles that are older than your last call. + Use the max_batches parameter to limit the number of batches/pages to retrieve. :param session: The authenticated session to use to make the api calls :param newer_than: Only articles newer than this parameter will be used. If None only one call will be make, and the continuation will be ignored + :param older_than: Only articles older than this parameter will be used. If None only one call will be make, + and the continuation will be ignored + :param max_batches: Maximum number of batches to retrieve. If None, will continue until no more data is available """ self.newer_than = newer_than + self.older_than = older_than self.session = session self.stream_id = stream_id + self.max_batches = max_batches def download_all(self) -> List[T]: return self._merge(self.stream_bundles()) def stream_bundles(self) -> Iterable[T]: continuation = None + batch_count = 0 while True: - resp = self.session.make_api_request( - f"{self.RELATIVE_URL}", - params={ - "newerThan": int(self.newer_than.timestamp()) if self.newer_than else None, - "continuation": continuation, - "streamId": self.stream_id, - "format": self.FORMAT, - }, - ) + params = { + "continuation": continuation, + "streamId": self.stream_id, + "format": self.FORMAT, + } + if self.newer_than: + params["newerThan"] = int(self.newer_than.timestamp()) + if self.older_than: + params["olderThan"] = int(self.older_than.timestamp()) + + resp = self.session.make_api_request(f"{self.RELATIVE_URL}", params=params) yield self._parse_response(resp) + batch_count += 1 + + # Check if we've reached max_batches limit + if self.max_batches and batch_count >= self.max_batches: + return + if not self.newer_than or "link" not in resp.headers: return next_url = resp.headers["link"][1:].split(">")[0] @@ -69,21 +93,35 @@ def _merge(self, resp_jsons: Iterable[T]) -> T: class IoCDownloaderBuilder: - def __init__(self, session: FeedlySession, format: IoCFormat, newer_than: Optional[datetime] = None): + def __init__( + self, + session: FeedlySession, + format: IoCFormat, + newer_than: Optional[datetime] = None, + older_than: Optional[datetime] = None, + max_batches: Optional[int] = None, + ): """ Use this class to export the contextualized IoCs from a stream. Enterprise/personals feeds/boards are supported (see dedicated methods below). The IoCs will be returned along with their context and relationships in a dictionary representing a valid STIX v2.1 Bundle object. https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_gms872kuzdmg Use the newer_than parameter to filter articles that are newer than your last call. + Use the older_than parameter to filter articles that are older than your last call. + Use the max_batches parameter to limit the number of batches/pages to retrieve. :param session: The authenticated session to use to make the api calls :param newer_than: Only articles newer than this parameter will be used. If None only one call will be make, and the continuation will be ignored + :param older_than: Only articles older than this parameter will be used. If None only one call will be make, + and the continuation will be ignored + :param max_batches: Maximum number of batches to retrieve. If None, will continue until no more data is available """ self.session = session self.format = format self.newer_than = newer_than + self.older_than = older_than + self.max_batches = max_batches self.session.api_host = "https://cloud.feedly.com" self.user = self.session.user @@ -107,8 +145,18 @@ def from_stream(self, stream: Streamable) -> IoCDownloaderABC: return self.from_stream_id(stream.id) def from_stream_id(self, stream_id: str) -> IoCDownloaderABC: - format2class = {IoCFormat.MISP: MispIoCDownloader, IoCFormat.STIX: StixIoCDownloader} - return format2class[self.format](session=self.session, newer_than=self.newer_than, stream_id=stream_id) + format2class = { + IoCFormat.MISP: MispIoCDownloader, + IoCFormat.STIX: StixIoCDownloader, + IoCFormat.CSV: CsvIoCDownloader, + } + return format2class[self.format]( + session=self.session, + newer_than=self.newer_than, + older_than=self.older_than, + stream_id=stream_id, + max_batches=self.max_batches, + ) class StixIoCDownloader(IoCDownloaderABC[Dict]): @@ -127,3 +175,13 @@ class MispIoCDownloader(IoCDownloaderABC[Dict]): def _merge(self, resp_jsons: Iterable[Dict]) -> Dict: return {"response": list(chain.from_iterable(resp_json["response"] for resp_json in resp_jsons))} + + +class CsvIoCDownloader(IoCDownloaderABC[List[Dict]]): + FORMAT = "csv" + + def _merge(self, resp_jsons: Iterable[List[Dict]]) -> List[Dict]: + return list(chain.from_iterable(resp_jsons)) + + def _parse_response(self, resp: Response) -> List[Dict]: + return list(DictReader(resp.text.splitlines())) \ No newline at end of file diff --git a/feedly/api_client/stream.py b/feedly/api_client/stream.py index 5aa1777..a06d95d 100644 --- a/feedly/api_client/stream.py +++ b/feedly/api_client/stream.py @@ -110,9 +110,9 @@ class StreamOptions: to produce url parameters """ - def __init__(self, max_count: int = 100): + def __init__(self, max_count: int = 100, ranked: str = "newest"): self.count: int = 20 - self.ranked: str = "newest" + self.ranked: str = ranked self.unreadOnly: bool = False self.newerThan: int = None self._max_count = max_count diff --git a/setup.py b/setup.py index d76ac94..bc25cfd 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ EMAIL = "ml@feedly.com" AUTHOR = "Feedly" REQUIRES_PYTHON = ">=3.6.0" -VERSION = "0.24" +VERSION = "0.27" # What packages are required for this module to be executed? with open("requirements.txt") as f: