Slides | AICell Lab

Slides

Fri, 05 Feb 2021 00:00:00 +0000

ImJoy: Supercharging Interactivity and Scalability for BioImage Analysis

Wei Ouyang

KTH | SciLifeLab, Stockholm

Challenges in AI for bioimaging

Usability: User friendly GUI
Flexibility: Flexible for different data types
Interactivity: Respond to GUI on laptop/mobile
Scalability: Remote storage and compute resources
Privacy: Edge computing

Progressive Web App

Rich and interactive UI libraries
Computation in the browser (+cloud)
Offline support

ImJoy https://imjoy.io

Data science tools in the browser

----- # Key concepts * Sandboxed plugins connected via Remote Procedure calls * Workflow composition via asynchronous programming * Open Integration with existing software/website

Calling Python from JS with RPC

Python (cloud) ⇔ Javascript (local)

-----

👐Open Integration with Web Apps

Customize annotation workflow with Kaibu

// load the web app via its URL
viewer = await api.createWindow({src: "https://kaibu.org/#/app"})
// call api functions directly via RPC
// add an image layer
await viewer.view_image("https://images.proteinatlas.org/61448/1319_C10_2_blue_red_green.jpg")
// add an annotation layer
await viewer.add_shapes([], {name:"annotation"})

🔥Demo: ImJoy + ImageJ => ImageJ.JS

🔥Demo: Visualization with Vizarr

Made by Trevor Manz et. al.

🚀A rapid growing list of plugins

ImageJ.JS
File Manager
Vizarr for visualizing zarr images
ITK/VTK Viewer for 3D visualizing
ImJoy Slides
ImJoy Chart Editor

Works with Jupyter/Binder and Colab

Advanced AI models in one click!

🤔How it works

🔥Try it yourself!

https://bioimage.io

BioEngine – AI model and application serving

Fetch models from bioimage.io
Provide web API for model training and inference
Support test run models and ImJoy applications
Suitable for deploying AI workflows for institutions, facilities or labs

Running models in JupyterLite via the BioEngine

AI-assisted Bioimage Analysis

A taste of the future!

🔥Codex live demos!

Disclaimer!

All the demo applications have not yet been approved for launch
Please don’t record the demos

Seminar at SciLifeLab

A sneak peek into the future of AI-assisted life science

Tomorrow (November 23rd), 10:00 - 11:00 CET
https://www.scilifelab.se/event/scilifelab-ai-seminar-series-wei-ouyang/

Conclusions

ImJoy is built for scalability and interactivity
ImJoy plugins are sandbox services connected via RPC
BioImage Model Zoo for model sharing
BioEninge for AI model serving
The future of bioimage anlysis

Acknowledgements

Work carried out at Cell Profiling group @ SciLifeLab headed by Emma Lundberg

BioImage.IO is powered by the 🧠 and ❤️ of:

deepImageJ Team
EBI Bioimage Archive Team
Fiji/ImageJ Team
ilastik Team
ImJoy Team
ZeroCostDL4Mic Team
…

Acknowledgements

We thank OpenAI for providing beta testing access to GPT-3 and Codex

The demos during this talk has not been approved by OpenAI

🙏Thank You!

const PythonPluginCode = `
<config lang="json">
{
 "name": "PythonPlugin",
 "type": "native-python",
 "version": "0.1.0",
 "description": "[TODO: describe this plugin with one sentence.]",
 "tags": [],
 "ui": "",
 "cover": "",
 "inputs": null,
 "outputs": null,
 "flags": [],
 "icon": "extension",
 "api_version": "0.1.8",
 "env": "",
 "permissions": [],
 "requirements": [],
 "dependencies": []
}
</config>

<script lang="python">
from imjoy import api


class ImJoyPlugin():
 def setup(self):
 api.showMessage('Python plugin initialized')

 def add(self, a, b):
 return a + b

api.export(ImJoyPlugin())
</script>
`

const JSPluginCode = `
<config lang="json">
{
 "name": "JSPlugin",
 "type": "window",
 "tags": [],
 "ui": "",
 "version": "0.1.0",
 "cover": "",
 "description": "[TODO: describe this plugin with one sentence.]",
 "icon": "extension",
 "inputs": null,
 "outputs": null,
 "api_version": "0.1.8",
 "env": "",
 "permissions": [],
 "requirements": [],
 "dependencies": [],
 "defaults": {"w": 20, "h": 10}
}
</config>

<script lang="javascript">
window.callPython = async function(){
 const pythonPlugin = await api.getPlugin('PythonPlugin')
 const result = await pythonPlugin.add(10, 99)
 document.getElementById("result").innerHTML = "10 + 99 =" + result
}

class ImJoyPlugin {
 async setup() {
 api.log('initialized')
 }

 async run(ctx) {
 }
}
api.export(new ImJoyPlugin())
</script>

<window lang="html">
 <div>
 <button class="button" onclick="callPython()"> Calculate in Python</button>
 <h3 id="result"></h3>
 </div>
</window>

<style lang="css">

</style>
`

window.ZarrPythonCode = `
<config lang="json">
{
 "name": "ZarrPythonPlugin",
 "type": "native-python",
 "version": "0.1.0",
 "description": "[TODO: describe this plugin with one sentence.]",
 "tags": [],
 "ui": "",
 "cover": "",
 "inputs": null,
 "outputs": null,
 "flags": [],
 "icon": "extension",
 "api_version": "0.1.8",
 "env": "",
 "permissions": [],
 "requirements": ["zarr", "fsspec"],
 "dependencies": []
}
</config>

<script lang="python">
import zarr
from imjoy_rpc import api
from imjoy_rpc import register_default_codecs
from fsspec.implementations.http import HTTPFileSystem
register_default_codecs()

fs = HTTPFileSystem()
http_map = fs.get_mapper("https://openimaging.github.io/demos/multi-scale-chunked-compressed/build/data/medium.zarr")
z_group = zarr.open(http_map, mode='r')

class ImJoyPlugin:
 async def setup(self):
 pass

 async def run(self, ctx):
 viewer = await api.createWindow(
 src="https://kitware.github.io/itk-vtk-viewer/app/",
 name="ITK/VTK Viewer"
 )
 await viewer.setImage(z_group)

api.export(ImJoyPlugin())
</script>
`
function startImageJ(){
 api.createWindow({src:"https://ij.imjoy.io", name:"ImageJ.JS"})
}

async function initializeMacroEditor(editor_container, code){
 const editorElm = document.getElementById(editor_container);
 if(!editorElm) throw new Error("editor container not found: " + editor_container)
 editorElm.style.width = '90%';
 editorElm.style.display = 'inline-block';
 editorElm.style.height = 'calc(100vh - 200px)';
 // force update the slide
 Reveal.layout();
 let editorWindow;
 const config = {lang: 'javascript'}
 config.templates = [
 {
 name: "New",
 url: null,
 lang: 'javascript',
 },
 {
 name: "Sphere",
 url: "https://wsr.imagej.net/download/Examples/Macro/Sphere.ijm",
 lang: 'javascript',
 },
 {
 name: "OpenDialog Demo",
 url: "https://wsr.imagej.net/download/Examples/Macro/OpenDialog_Demo.ijm",
 lang: 'javascript',
 },
 {
 name: "Overlay",
 url: "https://wsr.imagej.net/download/Examples/Macro/Overlay.ijm",
 lang: 'javascript',
 }
 ]
 config.ui_elements = {
 run: {
 _rintf: true,
 type: 'button',
 label: "Run",
 icon: "play",
 visible: true,
 shortcut: 'Shift-Enter',
 async callback(content) {
 try {
 let ij = await api.getWindow("ImageJ.JS-" + editor_container)
 if(!ij){
 //put the editor side by side
 editorElm.style.width = '38.2%';
 const ijElm = document.createElement('div');
 ijElm.id = 'imagej-' + editor_container
 ijElm.style.display = 'inline-block';
 ijElm.style.width = '61.8%';
 ijElm.style.height = editorElm.style.height;
 editorElm.parentNode.insertBefore(ijElm, editorElm.nextSibling);
 ij = await api.createWindow({src:"https://ij.imjoy.io", name:"ImageJ.JS-" + editor_container, window_id: 'imagej-' + editor_container})
 }
 await ij.runMacro(content)
 } catch (e) {
 api.showMessage("Failed to run macro, error: " + e.toString());
 } finally {
 editorWindow.updateUIElement('stop', {
 visible: false
 })
 editorWindow.setLoader(false);
 api.showProgress(100);
 }
 }
 },
 }
 editorWindow = await api.createWindow({
 src: 'https://if.imjoy.io',
 name: 'ImageJ Script Editor',
 config,
 window_id: editor_container,
 data: {code}
 })
}

Reveal.addEventListener('ij-macro-1', async ()=>{
 const code = `run("Blobs (25K)");
setAutoThreshold("Default");
setOption("BlackBackground", true);
run("Convert to Mask");
run("Analyze Particles...", "size=5-Infinity add");
`
 initializeMacroEditor('macro-editor-1', code)
})

Reveal.addEventListener('demo1', async function(){
 await api.createWindow({src: 'https://if.imjoy.io', config: {fold: [1]}, data: {code: PythonPluginCode}, window_id: "window-1"})

 await api.createWindow({src: 'https://if.imjoy.io', config: {fold: [1, 29]}, data: {code: JSPluginCode}, window_id: "window-2"})
})

async function runDemo2(){
 const viewer = await api.showDialog({src: "https://kaibu.org/#/app", name: "Kaibu"})
 await viewer.view_image("https://images.proteinatlas.org/61448/1319_C10_2_blue_red_green.jpg")
 await viewer.add_shapes([], {name:"annotation"})
}

Slides

Tue, 05 Feb 2019 00:00:00 +0000

EMBL-EBI Training - Microscopy data analysis

Wei OUYANG

SciLifeLab | KTH Royal Institute of Technology, Stockholm

https://bioimage.io

Challenges

Training models is difficult

Complex hardware/software setup
Environmental impact: “Training a single AI model can emit as much carbon as five cars in their lifetimes”

Lack of interoperability
Hard for non-expert to run

🦒BioImage Model Zoo

AI model file standard
Hosting pretrained models
Test run service
User/Developer services

🦒BioImage Model Zoo: Overview

🤔How it works

🔥Demos!

Overview: https://bioimage.io/
Find and download models
Upload Models
Ask for help

Meet the BioEngine

Deploying the BioEngine

BioEngine feature hightlights

Many models
Many users
Many applications
Shared GPU resources
Both inference and training
Local or cloud deployment

BioEngine vs Jupyter Notebooks / Colab

Scalability!

Cloud & On-premise deployment
For multi-user or the public
Multi-model serving
Improved GPU utilization
Instant usage without setup or installation

Accessing the BioEngine from Icy

Collabration with Carlos García López de Haro and the Icy Team

Accessing the BioEngine from Icy

Collabration with Carlos García López de Haro and the Icy Team

🔥Hands on tutorial

Try the BioEngine

Tutorial 1: Try the bioEngine

Go to https://bioimage.io
Search for 10.5281/zenodo.5869899
Click the BioEngine icon
Click input image to download an sample image
Click the select image button, drag and drop the image to elFinder
Select the uploaded image
Choose the pytorch weights and fill yx in the axes field
Click “Submit”
Wait for a while and see the result

Tutorial 2: Use the BioEnine via JupyterLite

Conclusions

AI model sharing via BioImage Model Zoo
BioEninge for scalable AI model serving

Acknowledgements (1)

Work carried out at Cell Profiling group @ SciLifeLab headed by Emma Lundberg

BioImage.IO is powered by the 🧠 and ❤️ of:

deepImageJ Team
EBI Bioimage Archive Team
Fiji/ImageJ Team
ilastik Team
ImJoy Team
ZeroCostDL4Mic Team
…

🙏Thank You!


async function loadNotebook(name, window_id, url, overwrite){
 const jupyter = await api.createWindow({src: "https://jupyter.imjoy.io/lab/index.html", window_id})
 const bid = window_id.replace("window", "reset")
 const button = document.getElementById(bid)
 if(await jupyter.fileExists(name)){
 if(overwrite){
 const content = await (await fetch(url)).text()
 await jupyter.removeFile(name)
 await jupyter.loadFile(name, content, 'application/json')
 }
 await jupyter.openFile(name)
 } else{
 const content = await (await fetch(url)).text()
 const filePath = await jupyter.loadFile(name, content, 'application/json')
 await jupyter.openFile(filePath)
 }
 button.style.display = "inline-block";
}


const PythonPluginCode = `
<config lang="json">
{
 "name": "PythonPlugin",
 "type": "native-python",
 "version": "0.1.0",
 "description": "[TODO: describe this plugin with one sentence.]",
 "tags": [],
 "ui": "",
 "cover": "",
 "inputs": null,
 "outputs": null,
 "flags": [],
 "icon": "extension",
 "api_version": "0.1.8",
 "env": "",
 "permissions": [],
 "requirements": [],
 "dependencies": []
}
</config>

<script lang="python">
from imjoy import api


class ImJoyPlugin():
 def setup(self):
 api.showMessage('Python plugin initialized')

 def add(self, a, b):
 return a + b

api.export(ImJoyPlugin())
</script>
`

const JSPluginCode = `
<config lang="json">
{
 "name": "JSPlugin",
 "type": "window",
 "tags": [],
 "ui": "",
 "version": "0.1.0",
 "cover": "",
 "description": "[TODO: describe this plugin with one sentence.]",
 "icon": "extension",
 "inputs": null,
 "outputs": null,
 "api_version": "0.1.8",
 "env": "",
 "permissions": [],
 "requirements": [],
 "dependencies": [],
 "defaults": {"w": 20, "h": 10}
}
</config>

<script lang="javascript">
window.callPython = async function(){
 const pythonPlugin = await api.getPlugin('PythonPlugin')
 const result = await pythonPlugin.add(10, 99)
 document.getElementById("result").innerHTML = "10 + 99 =" + result
}

class ImJoyPlugin {
 async setup() {
 api.log('initialized')
 }

 async run(ctx) {
 }
}
api.export(new ImJoyPlugin())
</script>

<window lang="html">
 <div>
 <button class="button" onclick="callPython()"> Calculate in Python</button>
 <h3 id="result"></h3>
 </div>
</window>
`

window.ZarrPythonCode = `
<config lang="json">
{
 "name": "ZarrPythonPlugin",
 "type": "native-python",
 "version": "0.1.0",
 "description": "[TODO: describe this plugin with one sentence.]",
 "tags": [],
 "ui": "",
 "cover": "",
 "inputs": null,
 "outputs": null,
 "flags": [],
 "icon": "extension",
 "api_version": "0.1.8",
 "env": "",
 "permissions": [],
 "requirements": ["zarr", "fsspec"],
 "dependencies": []
}
</config>

<script lang="python">
import zarr
from imjoy_rpc import api
from imjoy_rpc import register_default_codecs
from fsspec.implementations.http import HTTPFileSystem
register_default_codecs()

fs = HTTPFileSystem()
http_map = fs.get_mapper("https://openimaging.github.io/demos/multi-scale-chunked-compressed/build/data/medium.zarr")
z_group = zarr.open(http_map, mode='r')

class ImJoyPlugin:
 async def setup(self):
 pass

 async def run(self, ctx):
 viewer = await api.createWindow(
 src="https://kitware.github.io/itk-vtk-viewer/app/",
 name="ITK/VTK Viewer"
 )
 await viewer.setImage(z_group)

api.export(ImJoyPlugin())
</script>
`
function startImageJ(){
 api.createWindow({src:"https://ij.imjoy.io", name:"ImageJ.JS"})
}

async function initializeMacroEditor(editor_container, code){
 const editorElm = document.getElementById(editor_container);
 if(!editorElm) throw new Error("editor container not found: " + editor_container)
 editorElm.style.width = '90%';
 editorElm.style.display = 'inline-block';
 editorElm.style.height = 'calc(100vh - 200px)';
 // force update the slide
 Reveal.layout();
 let editorWindow;
 const config = {lang: 'javascript'}
 config.templates = [
 {
 name: "New",
 url: null,
 lang: 'javascript',
 },
 {
 name: "Sphere",
 url: "https://wsr.imagej.net/download/Examples/Macro/Sphere.ijm",
 lang: 'javascript',
 },
 {
 name: "OpenDialog Demo",
 url: "https://wsr.imagej.net/download/Examples/Macro/OpenDialog_Demo.ijm",
 lang: 'javascript',
 },
 {
 name: "Overlay",
 url: "https://wsr.imagej.net/download/Examples/Macro/Overlay.ijm",
 lang: 'javascript',
 }
 ]
 config.ui_elements = {
 run: {
 _rintf: true,
 type: 'button',
 label: "Run",
 icon: "play",
 visible: true,
 shortcut: 'Shift-Enter',
 async callback(content) {
 try {
 let ij = await api.getWindow("ImageJ.JS-" + editor_container)
 if(!ij){
 //put the editor side by side
 editorElm.style.width = '38.2%';
 const ijElm = document.createElement('div');
 ijElm.id = 'imagej-' + editor_container
 ijElm.style.display = 'inline-block';
 ijElm.style.width = '61.8%';
 ijElm.style.height = editorElm.style.height;
 editorElm.parentNode.insertBefore(ijElm, editorElm.nextSibling);
 ij = await api.createWindow({src:"https://ij.imjoy.io", name:"ImageJ.JS-" + editor_container, window_id: 'imagej-' + editor_container})
 }
 await ij.runMacro(content)
 } catch (e) {
 api.showMessage("Failed to run macro, error: " + e.toString());
 } finally {
 editorWindow.updateUIElement('stop', {
 visible: false
 })
 editorWindow.setLoader(false);
 api.showProgress(100);
 }
 }
 },
 }
 editorWindow = await api.createWindow({
 src: 'https://if.imjoy.io',
 name: 'ImageJ Script Editor',
 config,
 window_id: editor_container,
 data: {code}
 })
}

Reveal.addEventListener('ij-macro-1', async ()=>{
 const code = `run("Blobs (25K)");
setAutoThreshold("Default");
setOption("BlackBackground", true);
run("Convert to Mask");
run("Analyze Particles...", "size=5-Infinity add");
`
 initializeMacroEditor('macro-editor-1', code)
})

Reveal.addEventListener('demo1', async function(){
 await api.createWindow({src: 'https://if.imjoy.io', config: {fold: [1]}, data: {code: PythonPluginCode}, window_id: "window-1"})

 await api.createWindow({src: 'https://if.imjoy.io', config: {fold: [1, 29]}, data: {code: JSPluginCode}, window_id: "window-2"})
})

async function runDemo2(){
 const viewer = await api.showDialog({src: "https://kaibu.org/#/app", name: "Kaibu"})
 await viewer.view_image("https://images.proteinatlas.org/61448/1319_C10_2_blue_red_green.jpg")
 await viewer.add_shapes([], {name:"annotation"})
}

async function runHPADemo(){
 const plugin = await api.getPlugin("https://gist.githubusercontent.com/oeway/b318a26ef7191679b175be5216accbda/raw/HPA-UMAP-Studio.imjoy.html")
 await plugin.run({})
}


function startImageJ(){
 api.createWindow({src:"https://ij.imjoy.io", name:"ImageJ.JS"})
}

async function initializeMacroEditor(editor_container, code){
 const editorElm = document.getElementById(editor_container);
 if(!editorElm) throw new Error("editor container not found: " + editor_container)
 editorElm.style.width = '90%';
 editorElm.style.display = 'inline-block';
 editorElm.style.height = 'calc(100vh - 200px)';
 // force update the slide
 Reveal.layout();
 let editorWindow;
 const config = {lang: 'javascript'}
 config.templates = [
 {
 name: "New",
 url: null,
 lang: 'javascript',
 },
 {
 name: "Sphere",
 url: "https://wsr.imagej.net/download/Examples/Macro/Sphere.ijm",
 lang: 'javascript',
 },
 {
 name: "OpenDialog Demo",
 url: "https://wsr.imagej.net/download/Examples/Macro/OpenDialog_Demo.ijm",
 lang: 'javascript',
 },
 {
 name: "Overlay",
 url: "https://wsr.imagej.net/download/Examples/Macro/Overlay.ijm",
 lang: 'javascript',
 }
 ]
 config.ui_elements = {
 run: {
 _rintf: true,
 type: 'button',
 label: "Run",
 icon: "play",
 visible: true,
 shortcut: 'Shift-Enter',
 async callback(content) {
 try {
 let ij = await api.getWindow("ImageJ.JS-" + editor_container)
 if(!ij){
 //put the editor side by side
 editorElm.style.width = '38.2%';
 const ijElm = document.createElement('div');
 ijElm.id = 'imagej-' + editor_container
 ijElm.style.display = 'inline-block';
 ijElm.style.width = '61.8%';
 ijElm.style.height = editorElm.style.height;
 editorElm.parentNode.insertBefore(ijElm, editorElm.nextSibling);
 ij = await api.createWindow({src:"https://ij.imjoy.io", name:"ImageJ.JS-" + editor_container, window_id: 'imagej-' + editor_container})
 }
 await ij.runMacro(content)
 } catch (e) {
 api.showMessage("Failed to run macro, error: " + e.toString());
 } finally {
 editorWindow.updateUIElement('stop', {
 visible: false
 })
 editorWindow.setLoader(false);
 api.showProgress(100);
 }
 }
 },
 }
 editorWindow = await api.createWindow({
 src: 'https://if.imjoy.io',
 name: 'ImageJ Script Editor',
 config,
 window_id: editor_container,
 data: {code}
 })
}

Reveal.addEventListener('ij-macro-1', async ()=>{
 const code = `run("Blobs (25K)");
setAutoThreshold("Default");
setOption("BlackBackground", true);
run("Convert to Mask");
run("Analyze Particles...", "size=5-Infinity add");
`
 initializeMacroEditor('macro-editor-1', code)
})

Reveal.addEventListener('ij-macro-2', async ()=>{
 const response = await fetch("https://wsr.imagej.net/download/Examples/Macro/Colors_of_2021.ijm")
 const code = await response.text()
 initializeMacroEditor('macro-editor-2', code)
})