From 8faae94fed69db3df1c2daa2fb9840c2ebd6fda7 Mon Sep 17 00:00:00 2001 From: Michele Bologna Date: Mon, 24 Oct 2022 14:19:54 +0200 Subject: [PATCH 01/11] Feat: make use of ranked sorting when streaming --- feedly/api_client/stream.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/feedly/api_client/stream.py b/feedly/api_client/stream.py index 5aa1777..a06d95d 100644 --- a/feedly/api_client/stream.py +++ b/feedly/api_client/stream.py @@ -110,9 +110,9 @@ class StreamOptions: to produce url parameters """ - def __init__(self, max_count: int = 100): + def __init__(self, max_count: int = 100, ranked: str = "newest"): self.count: int = 20 - self.ranked: str = "newest" + self.ranked: str = ranked self.unreadOnly: bool = False self.newerThan: int = None self._max_count = max_count From 22cc8d00285a55bd17e738742e64730c57867c0e Mon Sep 17 00:00:00 2001 From: Mathieu Beligon Date: Tue, 25 Oct 2022 11:30:46 -0700 Subject: [PATCH 02/11] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d76ac94..13d040b 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ EMAIL = "ml@feedly.com" AUTHOR = "Feedly" REQUIRES_PYTHON = ">=3.6.0" -VERSION = "0.24" +VERSION = "0.25" # What packages are required for this module to be executed? with open("requirements.txt") as f: From 50415da0ca8cb027686841b391bc479f70f7b81d Mon Sep 17 00:00:00 2001 From: Mathieu Beligon Date: Tue, 22 Nov 2022 13:59:50 -0800 Subject: [PATCH 03/11] [IoC] Add CSV export format --- ...tors_of_compromise_from_a_stream_as_csv.py | 44 +++++++++++++++++++ .../enterprise/indicators_of_compromise.py | 19 +++++++- 2 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 examples/enterprise/export_indicators_of_compromise_from_a_stream_as_csv.py diff --git a/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_csv.py b/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_csv.py new file mode 100644 index 0000000..1e28680 --- /dev/null +++ b/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_csv.py @@ -0,0 +1,44 @@ +import json +from csv import DictWriter +from datetime import datetime, timedelta +from pathlib import Path +from pprint import pprint + +from feedly.api_client.enterprise.indicators_of_compromise import IoCDownloaderBuilder, IoCFormat +from feedly.api_client.session import FeedlySession +from feedly.api_client.utils import run_example + +RESULTS_DIR = Path(__file__).parent / "results" +RESULTS_DIR.mkdir(exist_ok=True) + + +def example_export_indicators_of_compromise_from_all_enterprise_feeds_as_csv(): + """ + This example will save a CSV file containing the contextualized IoCs that Leo extracted during the past 12 + hours in all your enterprise feeds. + """ + # Authenticate using the default auth directory + session = FeedlySession() + + # Create the CSV IoC downloader builder object, and limit it to 12 hours + # Usually newer_than will be the datetime of the last fetch + downloader_builder = IoCDownloaderBuilder( + session=session, newer_than=datetime.now() - timedelta(hours=12), format=IoCFormat.CSV + ) + + # Fetch the IoC from all the enterprise categories + # You can use a different method to get the iocs from you personal categories, personal or enterprise boards, + # or from specific categories/boards using their names or ids + downloader = downloader_builder.from_all_enterprise_categories() + iocs = downloader.download_all() + + # Save the IoCs in a CSV + with (RESULTS_DIR / "ioc_example.csv").open("w") as f: + writer = DictWriter(f, fieldnames=list(iocs[0].keys())) + writer.writerows(iocs) + + +if __name__ == "__main__": + # Will prompt for the token if missing, and launch the example above + # If a token expired error is raised, will prompt for a new token and restart the example + run_example(example_export_indicators_of_compromise_from_all_enterprise_feeds_as_csv) diff --git a/feedly/api_client/enterprise/indicators_of_compromise.py b/feedly/api_client/enterprise/indicators_of_compromise.py index 94b3b49..3526844 100644 --- a/feedly/api_client/enterprise/indicators_of_compromise.py +++ b/feedly/api_client/enterprise/indicators_of_compromise.py @@ -1,11 +1,11 @@ import uuid from abc import ABC, abstractmethod +from csv import DictReader from datetime import datetime from enum import Enum from itertools import chain from typing import ClassVar, Dict, Generic, Iterable, List, Optional, TypeVar from urllib.parse import parse_qs - from requests import Response from feedly.api_client.data import Streamable @@ -17,6 +17,7 @@ class IoCFormat(Enum): MISP = "misp" STIX = "stix2.1" + CSV = "csv" class IoCDownloaderABC(ABC, Generic[T]): @@ -107,7 +108,11 @@ def from_stream(self, stream: Streamable) -> IoCDownloaderABC: return self.from_stream_id(stream.id) def from_stream_id(self, stream_id: str) -> IoCDownloaderABC: - format2class = {IoCFormat.MISP: MispIoCDownloader, IoCFormat.STIX: StixIoCDownloader} + format2class = { + IoCFormat.MISP: MispIoCDownloader, + IoCFormat.STIX: StixIoCDownloader, + IoCFormat.CSV: CsvIoCDownloader, + } return format2class[self.format](session=self.session, newer_than=self.newer_than, stream_id=stream_id) @@ -127,3 +132,13 @@ class MispIoCDownloader(IoCDownloaderABC[Dict]): def _merge(self, resp_jsons: Iterable[Dict]) -> Dict: return {"response": list(chain.from_iterable(resp_json["response"] for resp_json in resp_jsons))} + + +class CsvIoCDownloader(IoCDownloaderABC[List[Dict]]): + FORMAT = "csv" + + def _merge(self, resp_jsons: Iterable[List[Dict]]) -> List[Dict]: + return list(chain.from_iterable(resp_jsons)) + + def _parse_response(self, resp: Response) -> List[Dict]: + return list(DictReader(resp.text.splitlines())) \ No newline at end of file From 192ea6613cbddcf83934c537472a4345aede6572 Mon Sep 17 00:00:00 2001 From: Mathieu Beligon Date: Mon, 28 Nov 2022 11:50:00 -0800 Subject: [PATCH 04/11] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 13d040b..1fa7f3f 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ EMAIL = "ml@feedly.com" AUTHOR = "Feedly" REQUIRES_PYTHON = ">=3.6.0" -VERSION = "0.25" +VERSION = "0.26" # What packages are required for this module to be executed? with open("requirements.txt") as f: From 0e754534ebf0fa287d95d3047a467971fd859733 Mon Sep 17 00:00:00 2001 From: Mathieu Beligon Date: Fri, 30 Jun 2023 16:39:35 +0200 Subject: [PATCH 05/11] [examples] Add warning for TI requirement to avoid confusion --- .../export_indicators_of_compromise_from_a_stream_as_csv.py | 2 ++ .../export_indicators_of_compromise_from_a_stream_as_stix.py | 2 ++ examples/enterprise/export_indicators_of_compromise_to_misp.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_csv.py b/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_csv.py index 1e28680..f3afe75 100644 --- a/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_csv.py +++ b/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_csv.py @@ -39,6 +39,8 @@ def example_export_indicators_of_compromise_from_all_enterprise_feeds_as_csv(): if __name__ == "__main__": + # Warning: This example requires the Threat Intelligence package to be enabled on your account + # Will prompt for the token if missing, and launch the example above # If a token expired error is raised, will prompt for a new token and restart the example run_example(example_export_indicators_of_compromise_from_all_enterprise_feeds_as_csv) diff --git a/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_stix.py b/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_stix.py index d93bc11..74b5d07 100644 --- a/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_stix.py +++ b/examples/enterprise/export_indicators_of_compromise_from_a_stream_as_stix.py @@ -40,6 +40,8 @@ def example_export_indicators_of_compromise_from_all_enterprise_feeds_as_stix(): if __name__ == "__main__": + # Warning: This example requires the Threat Intelligence package to be enabled on your account + # Will prompt for the token if missing, and launch the example above # If a token expired error is raised, will prompt for a new token and restart the example run_example(example_export_indicators_of_compromise_from_all_enterprise_feeds_as_stix) diff --git a/examples/enterprise/export_indicators_of_compromise_to_misp.py b/examples/enterprise/export_indicators_of_compromise_to_misp.py index e385e07..6594df5 100644 --- a/examples/enterprise/export_indicators_of_compromise_to_misp.py +++ b/examples/enterprise/export_indicators_of_compromise_to_misp.py @@ -38,6 +38,8 @@ def export_indicators_of_compromise_to_misp(): if __name__ == "__main__": + # Warning: This example requires the Threat Intelligence package to be enabled on your account + logging.basicConfig(level="INFO") filterwarnings("ignore") From a5b1ce4726321118b3573112f5022069f920309a Mon Sep 17 00:00:00 2001 From: Edouard Mehlman Date: Thu, 9 Oct 2025 17:59:34 -0400 Subject: [PATCH 06/11] older than iocs downloader --- .../enterprise/indicators_of_compromise.py | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/feedly/api_client/enterprise/indicators_of_compromise.py b/feedly/api_client/enterprise/indicators_of_compromise.py index 3526844..74b4492 100644 --- a/feedly/api_client/enterprise/indicators_of_compromise.py +++ b/feedly/api_client/enterprise/indicators_of_compromise.py @@ -24,19 +24,23 @@ class IoCDownloaderABC(ABC, Generic[T]): RELATIVE_URL = "/v3/enterprise/ioc" FORMAT: ClassVar[str] - def __init__(self, session: FeedlySession, newer_than: Optional[datetime], stream_id: str): + def __init__(self, session: FeedlySession, newer_than: Optional[datetime], older_than: Optional[datetime], stream_id: str): """ Use this class to export the contextualized IoCs from a stream. Enterprise/personals feeds/boards are supported (see dedicated methods below). The IoCs will be returned along with their context and relationships in a dictionary representing a valid STIX v2.1 Bundle object. https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_gms872kuzdmg Use the newer_than parameter to filter articles that are newer than your last call. + Use the older_than parameter to filter articles that are older than your last call. :param session: The authenticated session to use to make the api calls :param newer_than: Only articles newer than this parameter will be used. If None only one call will be make, and the continuation will be ignored + :param older_than: Only articles older than this parameter will be used. If None only one call will be make, + and the continuation will be ignored """ self.newer_than = newer_than + self.older_than = older_than self.session = session self.stream_id = stream_id @@ -46,17 +50,22 @@ def download_all(self) -> List[T]: def stream_bundles(self) -> Iterable[T]: continuation = None while True: + params = { + "continuation": continuation, + "streamId": self.stream_id, + "format": self.FORMAT, + } + if self.newer_than: + params["newerThan"] = int(self.newer_than.timestamp()) + if self.older_than: + params["olderThan"] = int(self.older_than.timestamp()) + resp = self.session.make_api_request( f"{self.RELATIVE_URL}", - params={ - "newerThan": int(self.newer_than.timestamp()) if self.newer_than else None, - "continuation": continuation, - "streamId": self.stream_id, - "format": self.FORMAT, - }, + params=params, ) yield self._parse_response(resp) - if not self.newer_than or "link" not in resp.headers: + if (not self.newer_than and not self.older_than) or "link" not in resp.headers: return next_url = resp.headers["link"][1:].split(">")[0] continuation = parse_qs(next_url)["continuation"][0] @@ -70,21 +79,25 @@ def _merge(self, resp_jsons: Iterable[T]) -> T: class IoCDownloaderBuilder: - def __init__(self, session: FeedlySession, format: IoCFormat, newer_than: Optional[datetime] = None): + def __init__(self, session: FeedlySession, format: IoCFormat, newer_than: Optional[datetime] = None, older_than: Optional[datetime] = None): """ Use this class to export the contextualized IoCs from a stream. Enterprise/personals feeds/boards are supported (see dedicated methods below). The IoCs will be returned along with their context and relationships in a dictionary representing a valid STIX v2.1 Bundle object. https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_gms872kuzdmg Use the newer_than parameter to filter articles that are newer than your last call. + Use the older_than parameter to filter articles that are older than your last call. :param session: The authenticated session to use to make the api calls :param newer_than: Only articles newer than this parameter will be used. If None only one call will be make, and the continuation will be ignored + :param older_than: Only articles older than this parameter will be used. If None only one call will be make, + and the continuation will be ignored """ self.session = session self.format = format self.newer_than = newer_than + self.older_than = older_than self.session.api_host = "https://cloud.feedly.com" self.user = self.session.user @@ -113,7 +126,7 @@ def from_stream_id(self, stream_id: str) -> IoCDownloaderABC: IoCFormat.STIX: StixIoCDownloader, IoCFormat.CSV: CsvIoCDownloader, } - return format2class[self.format](session=self.session, newer_than=self.newer_than, stream_id=stream_id) + return format2class[self.format](session=self.session, newer_than=self.newer_than, older_than=self.older_than, stream_id=stream_id) class StixIoCDownloader(IoCDownloaderABC[Dict]): From 63cfaf63eca9e18f9df00ecd4c69b98fe3044986 Mon Sep 17 00:00:00 2001 From: Edouard Mehlman Date: Thu, 9 Oct 2025 18:04:05 -0400 Subject: [PATCH 07/11] formatting --- .../enterprise/indicators_of_compromise.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/feedly/api_client/enterprise/indicators_of_compromise.py b/feedly/api_client/enterprise/indicators_of_compromise.py index 74b4492..835158f 100644 --- a/feedly/api_client/enterprise/indicators_of_compromise.py +++ b/feedly/api_client/enterprise/indicators_of_compromise.py @@ -4,9 +4,9 @@ from datetime import datetime from enum import Enum from itertools import chain +from requests import Response from typing import ClassVar, Dict, Generic, Iterable, List, Optional, TypeVar from urllib.parse import parse_qs -from requests import Response from feedly.api_client.data import Streamable from feedly.api_client.session import FeedlySession @@ -24,7 +24,13 @@ class IoCDownloaderABC(ABC, Generic[T]): RELATIVE_URL = "/v3/enterprise/ioc" FORMAT: ClassVar[str] - def __init__(self, session: FeedlySession, newer_than: Optional[datetime], older_than: Optional[datetime], stream_id: str): + def __init__( + self, + session: FeedlySession, + newer_than: Optional[datetime], + older_than: Optional[datetime], + stream_id: str, + ): """ Use this class to export the contextualized IoCs from a stream. Enterprise/personals feeds/boards are supported (see dedicated methods below). @@ -60,12 +66,11 @@ def stream_bundles(self) -> Iterable[T]: if self.older_than: params["olderThan"] = int(self.older_than.timestamp()) - resp = self.session.make_api_request( - f"{self.RELATIVE_URL}", - params=params, - ) + resp = self.session.make_api_request(f"{self.RELATIVE_URL}", params=params) yield self._parse_response(resp) - if (not self.newer_than and not self.older_than) or "link" not in resp.headers: + if ( + not self.newer_than and not self.older_than + ) or "link" not in resp.headers: return next_url = resp.headers["link"][1:].split(">")[0] continuation = parse_qs(next_url)["continuation"][0] @@ -126,7 +131,12 @@ def from_stream_id(self, stream_id: str) -> IoCDownloaderABC: IoCFormat.STIX: StixIoCDownloader, IoCFormat.CSV: CsvIoCDownloader, } - return format2class[self.format](session=self.session, newer_than=self.newer_than, older_than=self.older_than, stream_id=stream_id) + return format2class[self.format]( + session=self.session, + newer_than=self.newer_than, + older_than=self.older_than, + stream_id=stream_id, + ) class StixIoCDownloader(IoCDownloaderABC[Dict]): From b3f365bdf7540e4eecae5b4c140a63b1aff1820c Mon Sep 17 00:00:00 2001 From: Edouard Mehlman Date: Thu, 9 Oct 2025 18:52:12 -0400 Subject: [PATCH 08/11] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1fa7f3f..bc25cfd 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ EMAIL = "ml@feedly.com" AUTHOR = "Feedly" REQUIRES_PYTHON = ">=3.6.0" -VERSION = "0.26" +VERSION = "0.27" # What packages are required for this module to be executed? with open("requirements.txt") as f: From ecec1681652c03215ba216d909c761ff9c007165 Mon Sep 17 00:00:00 2001 From: Edouard Mehlman Date: Thu, 9 Oct 2025 19:38:47 -0400 Subject: [PATCH 09/11] add max batches as well --- .../enterprise/indicators_of_compromise.py | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/feedly/api_client/enterprise/indicators_of_compromise.py b/feedly/api_client/enterprise/indicators_of_compromise.py index 835158f..73691c9 100644 --- a/feedly/api_client/enterprise/indicators_of_compromise.py +++ b/feedly/api_client/enterprise/indicators_of_compromise.py @@ -30,6 +30,7 @@ def __init__( newer_than: Optional[datetime], older_than: Optional[datetime], stream_id: str, + max_batches: Optional[int] = None, ): """ Use this class to export the contextualized IoCs from a stream. @@ -38,23 +39,27 @@ def __init__( STIX v2.1 Bundle object. https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_gms872kuzdmg Use the newer_than parameter to filter articles that are newer than your last call. Use the older_than parameter to filter articles that are older than your last call. + Use the max_batches parameter to limit the number of batches/pages to retrieve. :param session: The authenticated session to use to make the api calls :param newer_than: Only articles newer than this parameter will be used. If None only one call will be make, and the continuation will be ignored :param older_than: Only articles older than this parameter will be used. If None only one call will be make, and the continuation will be ignored + :param max_batches: Maximum number of batches to retrieve. If None, will continue until no more data is available """ self.newer_than = newer_than self.older_than = older_than self.session = session self.stream_id = stream_id + self.max_batches = max_batches def download_all(self) -> List[T]: return self._merge(self.stream_bundles()) def stream_bundles(self) -> Iterable[T]: continuation = None + batch_count = 0 while True: params = { "continuation": continuation, @@ -68,6 +73,12 @@ def stream_bundles(self) -> Iterable[T]: resp = self.session.make_api_request(f"{self.RELATIVE_URL}", params=params) yield self._parse_response(resp) + batch_count += 1 + + # Check if we've reached max_batches limit + if self.max_batches and batch_count >= self.max_batches: + return + if ( not self.newer_than and not self.older_than ) or "link" not in resp.headers: @@ -84,7 +95,14 @@ def _merge(self, resp_jsons: Iterable[T]) -> T: class IoCDownloaderBuilder: - def __init__(self, session: FeedlySession, format: IoCFormat, newer_than: Optional[datetime] = None, older_than: Optional[datetime] = None): + def __init__( + self, + session: FeedlySession, + format: IoCFormat, + newer_than: Optional[datetime] = None, + older_than: Optional[datetime] = None, + max_batches: Optional[int] = None, + ): """ Use this class to export the contextualized IoCs from a stream. Enterprise/personals feeds/boards are supported (see dedicated methods below). @@ -92,17 +110,20 @@ def __init__(self, session: FeedlySession, format: IoCFormat, newer_than: Option STIX v2.1 Bundle object. https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_gms872kuzdmg Use the newer_than parameter to filter articles that are newer than your last call. Use the older_than parameter to filter articles that are older than your last call. + Use the max_batches parameter to limit the number of batches/pages to retrieve. :param session: The authenticated session to use to make the api calls :param newer_than: Only articles newer than this parameter will be used. If None only one call will be make, and the continuation will be ignored :param older_than: Only articles older than this parameter will be used. If None only one call will be make, and the continuation will be ignored + :param max_batches: Maximum number of batches to retrieve. If None, will continue until no more data is available """ self.session = session self.format = format self.newer_than = newer_than self.older_than = older_than + self.max_batches = max_batches self.session.api_host = "https://cloud.feedly.com" self.user = self.session.user @@ -136,6 +157,7 @@ def from_stream_id(self, stream_id: str) -> IoCDownloaderABC: newer_than=self.newer_than, older_than=self.older_than, stream_id=stream_id, + max_batches=self.max_batches, ) From 25a9ccb3f259849a2943bb05ba0956c1e77e4bc5 Mon Sep 17 00:00:00 2001 From: Edouard Mehlman Date: Fri, 10 Oct 2025 17:16:27 -0400 Subject: [PATCH 10/11] pr review --- feedly/api_client/enterprise/indicators_of_compromise.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/feedly/api_client/enterprise/indicators_of_compromise.py b/feedly/api_client/enterprise/indicators_of_compromise.py index 73691c9..8d2484d 100644 --- a/feedly/api_client/enterprise/indicators_of_compromise.py +++ b/feedly/api_client/enterprise/indicators_of_compromise.py @@ -79,9 +79,7 @@ def stream_bundles(self) -> Iterable[T]: if self.max_batches and batch_count >= self.max_batches: return - if ( - not self.newer_than and not self.older_than - ) or "link" not in resp.headers: + if not self.newer_than or "link" not in resp.headers: return next_url = resp.headers["link"][1:].split(">")[0] continuation = parse_qs(next_url)["continuation"][0] From 46642111cd6bfac65a5dca78c389f20d7c5a7ad7 Mon Sep 17 00:00:00 2001 From: Edouard Mehlman Date: Fri, 10 Oct 2025 17:52:40 -0400 Subject: [PATCH 11/11] publish yml --- .github/workflows/publish.yml | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..d0976cb --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,40 @@ +name: Publish to PyPI + +on: + push: + tags: + - 'v*' # Trigger on version tags like v1.0.0 + release: + types: [published] # Also trigger on GitHub releases + workflow_dispatch: # Allow manual triggering + +jobs: + build-and-publish: + runs-on: ubuntu-latest + + environment: + name: pypi + url: https://pypi.org/project/feedly-client/ + + permissions: + id-token: write # IMPORTANT: OIDC token generation + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + + - name: Build package + run: python -m build + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 +