Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 10 additions & 9 deletions docs-site/docs/docs/notebook-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,25 +56,26 @@ Note that this method will automatically cache all of the dependencies from the

#### Dependency Caching

By default, Polynote caches JVM dependencies that are specified with URLs, as well as the [virtual environment created
for your notebook](python.md#python-dependencies).
By default, Polynote caches JVM dependencies that are specified with HTTP/HTTPS URLs, as well as the [virtual environment created
for your notebook](python.md#python-dependencies).

You can choose to manually bust the cache by either appending `?nocache` to the end of the dependency, or by
unfolding the Advanced Options pane for your dependency by clicking on the `...` button next to it.
You can choose to manually bust the cache by either appending `?nocache` to the end of the dependency, or by
unfolding the Advanced Options pane for your dependency by clicking on the `...` button next to it.

![Dependency Caching](images/notebook-configuration-cache.png)

Changing the cache option affects different types of dependencies differently.
Changing the cache option affects different types of dependencies differently.

- **JVM Dependencies**
- URL-based dependencies are affected by this setting. If using the cache, Polynote uses the cached file (if
present) instead of downloading it again. Conversely, if the cache is disabled for this dependency then Polynote
- HTTP/HTTPS URL-based dependencies are affected by this setting. If using the cache, Polynote uses the cached file (if
present) instead of downloading it again. Conversely, if the cache is disabled for this dependency then Polynote
will download the jar anew each time.
- Other URL schemes (such as `s3://` or `file://`) are not affected by this setting and do not support caching.
- GAV notation dependencies are unaffected by this change (Coursier caches these dependencies itself and we don't
expose any way to change that for now)
- **Python Dependencies**
- Python dependencies are affected by this setting. Since they share a virtual environment for this notebook,
**bypassing the cache for any Python dependency will bust the cache for all Python dependencies**, since this is
- Python dependencies are affected by this setting. Since they share a virtual environment for this notebook,
**bypassing the cache for any Python dependency will bust the cache for all Python dependencies**, since this is
implmemented as a simple deletion and recreation of the virtual environment.

!!!question "Feedback requested"
Expand Down
40 changes: 36 additions & 4 deletions polynote-frontend/polynote/ui/component/notebook/notebookconfig.ts
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,21 @@ class Dependencies extends Disposable {
readonly el: TagElement<"div">;
private container: TagElement<"div">;

private isHttpUrl(dep: string): boolean {
try {
const url = new URL(dep);
return url.protocol === 'http:' || url.protocol === 'https:';
} catch {
return false;
}
}

private shouldSupportCaching(lang: string, dep: string): boolean {
const isPython = lang === 'python';
const isHttp = this.isHttpUrl(dep);
return isPython || isHttp;
}

constructor(dependenciesHandler: StateView<Record<string, string[]> | undefined>, stateHandler: StateHandler<NBConfig>) {
super()

Expand Down Expand Up @@ -185,15 +200,27 @@ class Dependencies extends Disposable {
private addDep(item?: DepRow["data"]) {
const data = item ?? {lang: this.defaultLang, dep: "", cache: true}

const updateAdvancedVisibility = () => {
const shouldShowCache = this.shouldSupportCaching(data.lang, data.dep);

if (shouldShowCache) {
detail.style.display = '';
} else {
detail.style.display = 'none';
row.classList.remove("show-advanced");
}
};

const type = dropdown(['dependency-type'], {scala: 'scala/jvm', python: 'pip'}, data.lang).change(evt => {
row.classList.remove(data.lang);
data.lang = type.options[type.selectedIndex].value;
row.classList.add(data.lang);

updateAdvancedVisibility();
});

const input = textbox(['dependency'], 'Dependency coordinate, URL, pip package', data.dep).change(evt => {
data.dep = input.value.trim()
data.dep = input.value.trim();
updateAdvancedVisibility();
});

const remove = iconButton(['remove'], 'Remove', 'minus-circle-red', 'Remove').click(evt => {
Expand All @@ -215,7 +242,7 @@ class Dependencies extends Disposable {
div([], [
para([], [
"Should Polynote use a cached version of this dependency, if available?",
" Applicable to URL or pip dependencies only."]),
" Applicable to HTTP/HTTPS URL or pip dependencies only."]),
para([], ["Note that if any pip dependency bypasses the cache, the entire virtual environment will be recreated."]),
cache
])
Expand All @@ -229,16 +256,21 @@ class Dependencies extends Disposable {
div(['dependency-row', 'notebook-config-row'], [type, input, detail, remove, add, advanced]),
{ data })
this.container.appendChild(row)
updateAdvancedVisibility();
}

get conf(): Record<string, string[]> {
return Array.from(this.container.children).reduce<Record<string, string[]>>((acc, row: DepRow) => {
if (row.data.dep) {
if (row.data.cache) {
// Remove ?nocache if present
row.data.dep = row.data.dep.endsWith("?nocache") ? row.data.dep.substr(0, row.data.dep.length - "?nocache".length) : row.data.dep;
} else {
} else if (this.shouldSupportCaching(row.data.lang, row.data.dep)) {
// Only add ?nocache for HTTP/HTTPS URLs or pip dependencies
row.data.dep = row.data.dep.endsWith("?nocache") ? row.data.dep : row.data.dep + "?nocache";
}
// For other URL types (s3, file, etc.), ignore cache setting and don't add ?nocache

acc[row.data.lang] = [...(acc[row.data.lang] || []), row.data.dep]
}
return acc
Expand Down