From dacca1807de663382d36d6a4a8fce540a3a3461e Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Mon, 9 Sep 2024 16:36:41 -0700 Subject: [PATCH 01/11] initial animated py panels --- .../SchemaIO/components/FrameLoaderView.tsx | 82 ++++++++ .../src/plugins/SchemaIO/components/index.ts | 1 + app/packages/embeddings/src/Testing.tsx | 196 ++++++++++++++++++ app/packages/embeddings/src/index.ts | 3 + app/packages/playback/package.json | 1 + app/yarn.lock | 6 +- fiftyone/operators/types.py | 14 ++ 7 files changed, 300 insertions(+), 3 deletions(-) create mode 100644 app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx create mode 100644 app/packages/embeddings/src/Testing.tsx diff --git a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx new file mode 100644 index 0000000000..7c0725b04b --- /dev/null +++ b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx @@ -0,0 +1,82 @@ +import React, { + forwardRef, + useCallback, + useEffect, + useMemo, + useState, +} from "react"; +import { ObjectSchemaType, ViewPropsType } from "../utils/types"; +import { + DEFAULT_FRAME_NUMBER, + GLOBAL_TIMELINE_ID, +} from "@fiftyone/playback/src/lib/constants"; +import { BufferRange } from "@fiftyone/utilities"; +import { usePanelEvent } from "@fiftyone/operators"; +import { + usePanelId, + usePanelState, + useSetPanelStateById, +} from "@fiftyone/spaces"; +import { useCreateTimeline } from "@fiftyone/playback/src/lib/use-create-timeline"; +import { Timeline } from "@mui/icons-material"; +import _ from "lodash"; + +export default function FrameLoaderView(props: ViewPropsType) { + const { schema, path, data } = props; + const { view = {} } = schema; + const { on_load_range, timeline_id, target } = view; + const { properties } = schema as ObjectSchemaType; + const panelId = usePanelId(); + const [myLocalFrameNumber, setMyLocalFrameNumber] = + React.useState(DEFAULT_FRAME_NUMBER); + const triggerEvent = usePanelEvent(); + const setPanelState = useSetPanelStateById(true); + const panelState = usePanelState(null, panelId, true); + + const loadRange = React.useCallback(async (range: BufferRange) => { + if (on_load_range) { + triggerEvent(panelId, { + params: { range }, + operator: on_load_range, + }); + } + }, []); + + const myRenderFrame = React.useCallback((frameNumber: number) => { + setMyLocalFrameNumber(frameNumber); + console.log("rendering frame", frameNumber, props); + setPanelState(panelId, (current) => { + const currentFrameData = data?.frames[frameNumber] || {}; + const currentData = current.data || {}; + const updatedData = { ...currentData }; + _.set(updatedData, target, currentFrameData); + return { ...current, data: updatedData }; + }); + }, []); + + const { isTimelineInitialized, subscribe } = useCreateTimeline({ + config: { + totalFrames: 50, + loop: true, + }, + }); + + useEffect(() => { + console.log("data", data); + }, [data]); + + React.useEffect(() => { + if (isTimelineInitialized) { + subscribe({ + name: timeline_id || GLOBAL_TIMELINE_ID, + subscription: { + id: "sub1", // hmmm + loadRange, + renderFrame: myRenderFrame, + }, + }); + } + }, [isTimelineInitialized, loadRange, myRenderFrame]); + + return null; +} diff --git a/app/packages/core/src/plugins/SchemaIO/components/index.ts b/app/packages/core/src/plugins/SchemaIO/components/index.ts index d837b31dfc..bb0fca6f6e 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/index.ts +++ b/app/packages/core/src/plugins/SchemaIO/components/index.ts @@ -48,3 +48,4 @@ export { default as TagsView } from "./TagsView"; export { default as TextFieldView } from "./TextFieldView"; export { default as TupleView } from "./TupleView"; export { default as UnsupportedView } from "./UnsupportedView"; +export { default as FrameLoaderView } from "./FrameLoaderView"; diff --git a/app/packages/embeddings/src/Testing.tsx b/app/packages/embeddings/src/Testing.tsx new file mode 100644 index 0000000000..6bda21ced3 --- /dev/null +++ b/app/packages/embeddings/src/Testing.tsx @@ -0,0 +1,196 @@ +export function main() { + registerComponent({ + name: "ComponentWithTimeline", + label: "ComponentWithTimeline", + component: ComponentWithTimeline, + type: PluginComponentType.Panel, + activator: () => true, + }); + registerComponent({ + name: "Component2WithTimeline", + label: "Component2WithTimeline", + component: Component2WithTimeline, + type: PluginComponentType.Panel, + activator: () => true, + }); +} + +import { BufferRange } from "@fiftyone/utilities"; +import React from "react"; +import { + DEFAULT_FRAME_NUMBER, + GLOBAL_TIMELINE_ID, + SEEK_BAR_DEBOUNCE, +} from "@fiftyone/playback/src/lib/constants"; +import { TimelineName } from "@fiftyone/playback/src/lib/state"; +import { useCreateTimeline } from "@fiftyone/playback/src/lib/use-create-timeline"; +import { useTimeline } from "@fiftyone/playback/src/lib/use-timeline"; +import { useTimelineVizUtils } from "@fiftyone/playback/src/lib/use-timeline-viz-utils"; +import { + FoTimelineContainer, + FoTimelineControlsContainer, + Playhead, + Seekbar, + SeekbarThumb, + Speed, + StatusIndicator, +} from "@fiftyone/playback/src/views/PlaybackElements"; +import { PluginComponentType, registerComponent } from "@fiftyone/plugins"; + +interface TimelineProps { + name?: TimelineName; + style?: React.CSSProperties; +} + +// the following is an example of how a timline component view can be created +export const Timeline = React.forwardRef( + ({ name: maybeTimelineName, style }, ref) => { + const name = maybeTimelineName ?? GLOBAL_TIMELINE_ID; + + const { frameNumber, playHeadState, config, play, pause } = + useTimeline(name); + + const { getSeekValue, seekTo } = useTimelineVizUtils(); + + const seekBarValue = React.useMemo(() => getSeekValue(), [frameNumber]); + + const onChangeSeek = React.useCallback( + (e: React.ChangeEvent) => { + const newSeekBarValue = Number(e.target.value); + seekTo(newSeekBarValue); + }, + [] + ); + + const [isHoveringSeekBar, setIsHoveringSeekBar] = React.useState(false); + + return ( + setIsHoveringSeekBar(true)} + onMouseLeave={() => setIsHoveringSeekBar(false)} + > + + + + + + + + + ); + } +); + +export const ComponentWithTimeline = () => { + const [myLocalFrameNumber, setMyLocalFrameNumber] = + React.useState(DEFAULT_FRAME_NUMBER); + + const loadRange = React.useCallback(async (range: BufferRange) => { + // no-op for now, but maybe for testing, i can resolve a promise inside settimeout + }, []); + + const myRenderFrame = React.useCallback((frameNumber: number) => { + setMyLocalFrameNumber(frameNumber); + }, []); + + const { isTimelineInitialized, subscribe } = useCreateTimeline({ + config: { + totalFrames: 50, + loop: true, + }, + }); + + React.useEffect(() => { + if (isTimelineInitialized) { + subscribe({ + name: GLOBAL_TIMELINE_ID, + subscription: { + id: "sub1", + loadRange, + renderFrame: myRenderFrame, + }, + }); + } + }, [isTimelineInitialized, loadRange, myRenderFrame]); + + if (!isTimelineInitialized) { + return
loading...
; + } + + return ( + <> +
+ creator frame number: {myLocalFrameNumber} +
+ + + ); +}; + +export const Component2WithTimeline = () => { + const [myLocalFrameNumber, setMyLocalFrameNumber] = + React.useState(DEFAULT_FRAME_NUMBER); + + const loadRange = React.useCallback(async (range: BufferRange) => { + console.log("loading range", range); + // no-op for now, but maybe for testing, i can resolve a promise inside settimeout + return new Promise((resolve) => { + setTimeout(() => { + console.log("resolved"); + resolve(); + }, 1000); + }); + }, []); + + const myRenderFrame = React.useCallback((frameNumber: number) => { + setMyLocalFrameNumber(frameNumber); + }, []); + + const { subscribe, isTimelineInitialized } = useTimeline(); + + React.useEffect(() => { + if (!isTimelineInitialized) { + return; + } + + subscribe({ + name: GLOBAL_TIMELINE_ID, + subscription: { + id: "sub3", + loadRange, + renderFrame: myRenderFrame, + }, + }); + }, [loadRange, myRenderFrame, isTimelineInitialized]); + + if (!isTimelineInitialized) { + return
loading...
; + } + + return ( + <> +
+ subscriber frame number: {myLocalFrameNumber} +
+ + + ); +}; diff --git a/app/packages/embeddings/src/index.ts b/app/packages/embeddings/src/index.ts index 29c0006e33..86ae1050c9 100644 --- a/app/packages/embeddings/src/index.ts +++ b/app/packages/embeddings/src/index.ts @@ -18,3 +18,6 @@ registerComponent({ }); // registerOperator(new OpenEmbeddingsPanel()); +import { main } from "./Testing"; + +main(); diff --git a/app/packages/playback/package.json b/app/packages/playback/package.json index 25a8cb4766..2bfbef3378 100644 --- a/app/packages/playback/package.json +++ b/app/packages/playback/package.json @@ -1,5 +1,6 @@ { "name": "@fiftyone/playback", + "main": "./src/index.ts", "packageManager": "yarn@3.2.1", "devDependencies": { "@eslint/compat": "^1.1.1", diff --git a/app/yarn.lock b/app/yarn.lock index 3e82586cb0..dc9917db4b 100644 --- a/app/yarn.lock +++ b/app/yarn.lock @@ -13869,13 +13869,13 @@ __metadata: linkType: hard "postcss@npm:^8.4.43": - version: 8.4.44 - resolution: "postcss@npm:8.4.44" + version: 8.4.45 + resolution: "postcss@npm:8.4.45" dependencies: nanoid: ^3.3.7 picocolors: ^1.0.1 source-map-js: ^1.2.0 - checksum: 64d9ce78253696bb64e608a54b362c9ddb537d3b38b58223ebce8260d6110d4e798ef1b3d57d8c28131417d9809187fd51d5c4263113536363444f8635e11bdb + checksum: 3223cdad4a9392c0b334ee3ee7e4e8041c631cb6160609cef83c18d2b2580e931dd8068ab13cc6000c1a254d57492ac6c38717efc397c5dcc9756d06bc9c44f3 languageName: node linkType: hard diff --git a/fiftyone/operators/types.py b/fiftyone/operators/types.py index fd69f6cd62..a22bd01230 100644 --- a/fiftyone/operators/types.py +++ b/fiftyone/operators/types.py @@ -2399,6 +2399,20 @@ def to_json(self): } +class FrameLoaderView(View): + """Utility for loading frames and animated panels. + + Args: + timeline_id (None): the ID of the timeline to load + on_load (None): the operator to execute when the frame is loaded + on_error (None): the operator to execute when the frame fails to load + on_load_range (None): the operator to execute when the frame is loading + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + class Container(BaseType): """Represents a base container for a container types.""" From fe7030b82a7995bc01684fc78f3f867c6871e09c Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Sun, 15 Sep 2024 18:08:36 -0700 Subject: [PATCH 02/11] frameloaderview fixes --- .../plugins/SchemaIO/components/FrameLoaderView.tsx | 2 ++ app/packages/embeddings/src/Testing.tsx | 13 +++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx index 7c0725b04b..9e2e4aa587 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx @@ -49,7 +49,9 @@ export default function FrameLoaderView(props: ViewPropsType) { const currentFrameData = data?.frames[frameNumber] || {}; const currentData = current.data || {}; const updatedData = { ...currentData }; + console.log("currentData", currentData); _.set(updatedData, target, currentFrameData); + console.log("updatedData", updatedData); return { ...current, data: updatedData }; }); }, []); diff --git a/app/packages/embeddings/src/Testing.tsx b/app/packages/embeddings/src/Testing.tsx index 6bda21ced3..24587f3076 100644 --- a/app/packages/embeddings/src/Testing.tsx +++ b/app/packages/embeddings/src/Testing.tsx @@ -5,6 +5,9 @@ export function main() { component: ComponentWithTimeline, type: PluginComponentType.Panel, activator: () => true, + panelOptions: { + surfaces: 'modal' + } }); registerComponent({ name: "Component2WithTimeline", @@ -112,6 +115,7 @@ export const ComponentWithTimeline = () => { }, []); const { isTimelineInitialized, subscribe } = useCreateTimeline({ + name: GLOBAL_TIMELINE_ID, config: { totalFrames: 50, loop: true, @@ -121,12 +125,9 @@ export const ComponentWithTimeline = () => { React.useEffect(() => { if (isTimelineInitialized) { subscribe({ - name: GLOBAL_TIMELINE_ID, - subscription: { - id: "sub1", - loadRange, - renderFrame: myRenderFrame, - }, + id: "sub1", + loadRange, + renderFrame: myRenderFrame, }); } }, [isTimelineInitialized, loadRange, myRenderFrame]); From bb166bb9826fb4fd8f935bb01f965bdf47a8b8bf Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Mon, 16 Sep 2024 12:24:07 -0700 Subject: [PATCH 03/11] py panels timeline fixes --- .../SchemaIO/components/FrameLoaderView.tsx | 7 +- app/packages/embeddings/src/Testing.tsx | 173 ++++++++---------- 2 files changed, 82 insertions(+), 98 deletions(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx index 9e2e4aa587..925e639f86 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx @@ -42,6 +42,8 @@ export default function FrameLoaderView(props: ViewPropsType) { } }, []); + const [currentFrame, setCurrentFrame] = useState(DEFAULT_FRAME_NUMBER); + const myRenderFrame = React.useCallback((frameNumber: number) => { setMyLocalFrameNumber(frameNumber); console.log("rendering frame", frameNumber, props); @@ -54,6 +56,7 @@ export default function FrameLoaderView(props: ViewPropsType) { console.log("updatedData", updatedData); return { ...current, data: updatedData }; }); + setCurrentFrame(frameNumber) }, []); const { isTimelineInitialized, subscribe } = useCreateTimeline({ @@ -80,5 +83,7 @@ export default function FrameLoaderView(props: ViewPropsType) { } }, [isTimelineInitialized, loadRange, myRenderFrame]); - return null; + return ( +

{currentFrame}

+ ) } diff --git a/app/packages/embeddings/src/Testing.tsx b/app/packages/embeddings/src/Testing.tsx index 24587f3076..de80847e96 100644 --- a/app/packages/embeddings/src/Testing.tsx +++ b/app/packages/embeddings/src/Testing.tsx @@ -1,20 +1,13 @@ export function main() { registerComponent({ - name: "ComponentWithTimeline", - label: "ComponentWithTimeline", - component: ComponentWithTimeline, + name: "TimelineCreator", + label: "TimelineCreator", + component: TimelineCreator, type: PluginComponentType.Panel, activator: () => true, panelOptions: { - surfaces: 'modal' - } - }); - registerComponent({ - name: "Component2WithTimeline", - label: "Component2WithTimeline", - component: Component2WithTimeline, - type: PluginComponentType.Panel, - activator: () => true, + surfaces: "modal", + }, }); } @@ -39,83 +32,34 @@ import { StatusIndicator, } from "@fiftyone/playback/src/views/PlaybackElements"; import { PluginComponentType, registerComponent } from "@fiftyone/plugins"; +import { useDefaultTimelineName } from "@fiftyone/playback/src/lib/use-default-timeline-name"; interface TimelineProps { name?: TimelineName; style?: React.CSSProperties; } - -// the following is an example of how a timline component view can be created -export const Timeline = React.forwardRef( - ({ name: maybeTimelineName, style }, ref) => { - const name = maybeTimelineName ?? GLOBAL_TIMELINE_ID; - - const { frameNumber, playHeadState, config, play, pause } = - useTimeline(name); - - const { getSeekValue, seekTo } = useTimelineVizUtils(); - - const seekBarValue = React.useMemo(() => getSeekValue(), [frameNumber]); - - const onChangeSeek = React.useCallback( - (e: React.ChangeEvent) => { - const newSeekBarValue = Number(e.target.value); - seekTo(newSeekBarValue); - }, - [] - ); - - const [isHoveringSeekBar, setIsHoveringSeekBar] = React.useState(false); - - return ( - setIsHoveringSeekBar(true)} - onMouseLeave={() => setIsHoveringSeekBar(false)} - > - - - - - - - - - ); - } -); - -export const ComponentWithTimeline = () => { +export const TimelineCreator = () => { const [myLocalFrameNumber, setMyLocalFrameNumber] = React.useState(DEFAULT_FRAME_NUMBER); + const { getName } = useDefaultTimelineName(); + const timelineName = React.useMemo(() => getName(), [getName]); const loadRange = React.useCallback(async (range: BufferRange) => { - // no-op for now, but maybe for testing, i can resolve a promise inside settimeout + return new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, 100); + }); }, []); - const myRenderFrame = React.useCallback((frameNumber: number) => { - setMyLocalFrameNumber(frameNumber); - }, []); + const myRenderFrame = React.useCallback( + (frameNumber: number) => { + setMyLocalFrameNumber(frameNumber); + }, + [setMyLocalFrameNumber] + ); const { isTimelineInitialized, subscribe } = useCreateTimeline({ - name: GLOBAL_TIMELINE_ID, config: { totalFrames: 50, loop: true, @@ -125,40 +69,79 @@ export const ComponentWithTimeline = () => { React.useEffect(() => { if (isTimelineInitialized) { subscribe({ - id: "sub1", + id: `creator`, loadRange, renderFrame: myRenderFrame, }); } - }, [isTimelineInitialized, loadRange, myRenderFrame]); + }, [isTimelineInitialized, loadRange, myRenderFrame, subscribe]); if (!isTimelineInitialized) { - return
loading...
; + return
initializing timeline...
; } return ( <>
- creator frame number: {myLocalFrameNumber} + creator frame number {timelineName}: {myLocalFrameNumber}
- + ); }; -export const Component2WithTimeline = () => { +export const TimelineSubscriber1 = () => { + const { getName } = useDefaultTimelineName(); + const timelineName = React.useMemo(() => getName(), [getName]); + const [myLocalFrameNumber, setMyLocalFrameNumber] = React.useState(DEFAULT_FRAME_NUMBER); const loadRange = React.useCallback(async (range: BufferRange) => { - console.log("loading range", range); // no-op for now, but maybe for testing, i can resolve a promise inside settimeout - return new Promise((resolve) => { - setTimeout(() => { - console.log("resolved"); - resolve(); - }, 1000); + }, []); + + const myRenderFrame = React.useCallback((frameNumber: number) => { + setMyLocalFrameNumber(frameNumber); + }, []); + + const { subscribe, isTimelineInitialized, getFrameNumber } = useTimeline(); + + React.useEffect(() => { + if (!isTimelineInitialized) { + return; + } + + subscribe({ + id: `sub1`, + loadRange, + renderFrame: myRenderFrame, }); + }, [loadRange, myRenderFrame, subscribe, isTimelineInitialized]); + + if (!isTimelineInitialized) { + return
loading...
; + } + + return ( + <> +
+ Subscriber 1 frame number {timelineName}: {myLocalFrameNumber} +
+ + + ); +}; + +export const TimelineSubscriber2 = () => { + const { getName } = useDefaultTimelineName(); + const timelineName = React.useMemo(() => getName(), [getName]); + + const [myLocalFrameNumber, setMyLocalFrameNumber] = + React.useState(DEFAULT_FRAME_NUMBER); + + const loadRange = React.useCallback(async (range: BufferRange) => { + // no-op for now, but maybe for testing, i can resolve a promise inside settimeout }, []); const myRenderFrame = React.useCallback((frameNumber: number) => { @@ -173,14 +156,11 @@ export const Component2WithTimeline = () => { } subscribe({ - name: GLOBAL_TIMELINE_ID, - subscription: { - id: "sub3", - loadRange, - renderFrame: myRenderFrame, - }, + id: `sub2`, + loadRange, + renderFrame: myRenderFrame, }); - }, [loadRange, myRenderFrame, isTimelineInitialized]); + }, [loadRange, myRenderFrame, subscribe, isTimelineInitialized]); if (!isTimelineInitialized) { return
loading...
; @@ -189,9 +169,8 @@ export const Component2WithTimeline = () => { return ( <>
- subscriber frame number: {myLocalFrameNumber} + Subscriber 2 frame number {timelineName}: {myLocalFrameNumber}
- ); }; From cadb2fb9521ef36608924e8a28bb38c2695cce1b Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Mon, 16 Sep 2024 14:22:39 -0700 Subject: [PATCH 04/11] more timeline fixes for py panels --- .../SchemaIO/components/FrameLoaderView.tsx | 51 +++++++++---------- app/packages/embeddings/src/Testing.tsx | 1 + 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx index 925e639f86..dff4f3f70a 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx @@ -17,9 +17,9 @@ import { usePanelState, useSetPanelStateById, } from "@fiftyone/spaces"; -import { useCreateTimeline } from "@fiftyone/playback/src/lib/use-create-timeline"; -import { Timeline } from "@mui/icons-material"; +import { useTimeline } from "@fiftyone/playback/src/lib/use-timeline"; import _ from "lodash"; +import { useDefaultTimelineName } from "@fiftyone/playback/src/lib/use-default-timeline-name"; export default function FrameLoaderView(props: ViewPropsType) { const { schema, path, data } = props; @@ -31,57 +31,54 @@ export default function FrameLoaderView(props: ViewPropsType) { React.useState(DEFAULT_FRAME_NUMBER); const triggerEvent = usePanelEvent(); const setPanelState = useSetPanelStateById(true); - const panelState = usePanelState(null, panelId, true); + const { getName } = useDefaultTimelineName(); + const timelineName = React.useMemo(() => getName(), [getName]); const loadRange = React.useCallback(async (range: BufferRange) => { + console.log("loadRange", range); if (on_load_range) { - triggerEvent(panelId, { + return triggerEvent(panelId, { params: { range }, operator: on_load_range, }); } + }, [triggerEvent, on_load_range]); + + useEffect(() => { + loadRange([0, 50]); }, []); const [currentFrame, setCurrentFrame] = useState(DEFAULT_FRAME_NUMBER); const myRenderFrame = React.useCallback((frameNumber: number) => { setMyLocalFrameNumber(frameNumber); - console.log("rendering frame", frameNumber, props); + // console.log("rendering frame", frameNumber, props); setPanelState(panelId, (current) => { const currentFrameData = data?.frames[frameNumber] || {}; - const currentData = current.data || {}; - const updatedData = { ...currentData }; - console.log("currentData", currentData); - _.set(updatedData, target, currentFrameData); + const currentData = current.data ? _.cloneDeep(current.data) : {}; // Clone the object + let updatedData = { ...currentData }; + + console.log("data?.frames", data?.frames); + console.log("target", target); + _.set(updatedData, target, currentFrameData); // Use lodash set to update safely console.log("updatedData", updatedData); + return { ...current, data: updatedData }; }); setCurrentFrame(frameNumber) - }, []); + }, [data, setPanelState, panelId, target]); - const { isTimelineInitialized, subscribe } = useCreateTimeline({ - config: { - totalFrames: 50, - loop: true, - }, - }); - - useEffect(() => { - console.log("data", data); - }, [data]); + const { isTimelineInitialized, subscribe } = useTimeline(); React.useEffect(() => { if (isTimelineInitialized) { subscribe({ - name: timeline_id || GLOBAL_TIMELINE_ID, - subscription: { - id: "sub1", // hmmm - loadRange, - renderFrame: myRenderFrame, - }, + id: `sub1`, + loadRange, + renderFrame: myRenderFrame, }); } - }, [isTimelineInitialized, loadRange, myRenderFrame]); + }, [isTimelineInitialized, loadRange, myRenderFrame, subscribe]); return (

{currentFrame}

diff --git a/app/packages/embeddings/src/Testing.tsx b/app/packages/embeddings/src/Testing.tsx index de80847e96..d9e2e146c5 100644 --- a/app/packages/embeddings/src/Testing.tsx +++ b/app/packages/embeddings/src/Testing.tsx @@ -33,6 +33,7 @@ import { } from "@fiftyone/playback/src/views/PlaybackElements"; import { PluginComponentType, registerComponent } from "@fiftyone/plugins"; import { useDefaultTimelineName } from "@fiftyone/playback/src/lib/use-default-timeline-name"; +import {Timeline} from "@fiftyone/playback/src/views/Timeline"; interface TimelineProps { name?: TimelineName; From 3d1a25fc1b5cca932c7c504bb405f801524383f2 Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Mon, 16 Sep 2024 14:28:34 -0700 Subject: [PATCH 05/11] cleanup from timeline api changes --- .../SchemaIO/components/FrameLoaderView.tsx | 70 ++++----- app/packages/embeddings/src/Testing.tsx | 138 +----------------- 2 files changed, 38 insertions(+), 170 deletions(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx index dff4f3f70a..dafdab5433 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx @@ -31,42 +31,46 @@ export default function FrameLoaderView(props: ViewPropsType) { React.useState(DEFAULT_FRAME_NUMBER); const triggerEvent = usePanelEvent(); const setPanelState = useSetPanelStateById(true); - const { getName } = useDefaultTimelineName(); - const timelineName = React.useMemo(() => getName(), [getName]); - const loadRange = React.useCallback(async (range: BufferRange) => { - console.log("loadRange", range); - if (on_load_range) { - return triggerEvent(panelId, { - params: { range }, - operator: on_load_range, - }); - } - }, [triggerEvent, on_load_range]); + const loadRange = React.useCallback( + async (range: BufferRange) => { + console.log("loadRange", range); + if (on_load_range) { + return triggerEvent(panelId, { + params: { range }, + operator: on_load_range, + }); + } + }, + [triggerEvent, on_load_range] + ); - useEffect(() => { - loadRange([0, 50]); - }, []); + // useEffect(() => { + // loadRange([0, 50]); + // }, []); const [currentFrame, setCurrentFrame] = useState(DEFAULT_FRAME_NUMBER); - const myRenderFrame = React.useCallback((frameNumber: number) => { - setMyLocalFrameNumber(frameNumber); - // console.log("rendering frame", frameNumber, props); - setPanelState(panelId, (current) => { - const currentFrameData = data?.frames[frameNumber] || {}; - const currentData = current.data ? _.cloneDeep(current.data) : {}; // Clone the object - let updatedData = { ...currentData }; - - console.log("data?.frames", data?.frames); - console.log("target", target); - _.set(updatedData, target, currentFrameData); // Use lodash set to update safely - console.log("updatedData", updatedData); - - return { ...current, data: updatedData }; - }); - setCurrentFrame(frameNumber) - }, [data, setPanelState, panelId, target]); + const myRenderFrame = React.useCallback( + (frameNumber: number) => { + setMyLocalFrameNumber(frameNumber); + // console.log("rendering frame", frameNumber, props); + setPanelState(panelId, (current) => { + const currentFrameData = data?.frames[frameNumber] || {}; + const currentData = current.data ? _.cloneDeep(current.data) : {}; // Clone the object + let updatedData = { ...currentData }; + + console.log("data?.frames", data?.frames); + console.log("target", target); + _.set(updatedData, target, currentFrameData); // Use lodash set to update safely + console.log("updatedData", updatedData); + + return { ...current, data: updatedData }; + }); + setCurrentFrame(frameNumber); + }, + [data, setPanelState, panelId, target] + ); const { isTimelineInitialized, subscribe } = useTimeline(); @@ -80,7 +84,5 @@ export default function FrameLoaderView(props: ViewPropsType) { } }, [isTimelineInitialized, loadRange, myRenderFrame, subscribe]); - return ( -

{currentFrame}

- ) + return

{currentFrame}

; } diff --git a/app/packages/embeddings/src/Testing.tsx b/app/packages/embeddings/src/Testing.tsx index d9e2e146c5..da3b9572f3 100644 --- a/app/packages/embeddings/src/Testing.tsx +++ b/app/packages/embeddings/src/Testing.tsx @@ -33,145 +33,11 @@ import { } from "@fiftyone/playback/src/views/PlaybackElements"; import { PluginComponentType, registerComponent } from "@fiftyone/plugins"; import { useDefaultTimelineName } from "@fiftyone/playback/src/lib/use-default-timeline-name"; -import {Timeline} from "@fiftyone/playback/src/views/Timeline"; +import { Timeline } from "@fiftyone/playback/src/views/Timeline"; interface TimelineProps { name?: TimelineName; style?: React.CSSProperties; } -export const TimelineCreator = () => { - const [myLocalFrameNumber, setMyLocalFrameNumber] = - React.useState(DEFAULT_FRAME_NUMBER); - const { getName } = useDefaultTimelineName(); - const timelineName = React.useMemo(() => getName(), [getName]); - const loadRange = React.useCallback(async (range: BufferRange) => { - return new Promise((resolve) => { - setTimeout(() => { - resolve(); - }, 100); - }); - }, []); - - const myRenderFrame = React.useCallback( - (frameNumber: number) => { - setMyLocalFrameNumber(frameNumber); - }, - [setMyLocalFrameNumber] - ); - - const { isTimelineInitialized, subscribe } = useCreateTimeline({ - config: { - totalFrames: 50, - loop: true, - }, - }); - - React.useEffect(() => { - if (isTimelineInitialized) { - subscribe({ - id: `creator`, - loadRange, - renderFrame: myRenderFrame, - }); - } - }, [isTimelineInitialized, loadRange, myRenderFrame, subscribe]); - - if (!isTimelineInitialized) { - return
initializing timeline...
; - } - - return ( - <> -
- creator frame number {timelineName}: {myLocalFrameNumber} -
- - - ); -}; - -export const TimelineSubscriber1 = () => { - const { getName } = useDefaultTimelineName(); - const timelineName = React.useMemo(() => getName(), [getName]); - - const [myLocalFrameNumber, setMyLocalFrameNumber] = - React.useState(DEFAULT_FRAME_NUMBER); - - const loadRange = React.useCallback(async (range: BufferRange) => { - // no-op for now, but maybe for testing, i can resolve a promise inside settimeout - }, []); - - const myRenderFrame = React.useCallback((frameNumber: number) => { - setMyLocalFrameNumber(frameNumber); - }, []); - - const { subscribe, isTimelineInitialized, getFrameNumber } = useTimeline(); - - React.useEffect(() => { - if (!isTimelineInitialized) { - return; - } - - subscribe({ - id: `sub1`, - loadRange, - renderFrame: myRenderFrame, - }); - }, [loadRange, myRenderFrame, subscribe, isTimelineInitialized]); - - if (!isTimelineInitialized) { - return
loading...
; - } - - return ( - <> -
- Subscriber 1 frame number {timelineName}: {myLocalFrameNumber} -
- - - ); -}; - -export const TimelineSubscriber2 = () => { - const { getName } = useDefaultTimelineName(); - const timelineName = React.useMemo(() => getName(), [getName]); - - const [myLocalFrameNumber, setMyLocalFrameNumber] = - React.useState(DEFAULT_FRAME_NUMBER); - - const loadRange = React.useCallback(async (range: BufferRange) => { - // no-op for now, but maybe for testing, i can resolve a promise inside settimeout - }, []); - - const myRenderFrame = React.useCallback((frameNumber: number) => { - setMyLocalFrameNumber(frameNumber); - }, []); - - const { subscribe, isTimelineInitialized } = useTimeline(); - - React.useEffect(() => { - if (!isTimelineInitialized) { - return; - } - - subscribe({ - id: `sub2`, - loadRange, - renderFrame: myRenderFrame, - }); - }, [loadRange, myRenderFrame, subscribe, isTimelineInitialized]); - - if (!isTimelineInitialized) { - return
loading...
; - } - - return ( - <> -
- Subscriber 2 frame number {timelineName}: {myLocalFrameNumber} -
- - ); -}; +import { TimelineCreator } from "@fiftyone/playback/src/views/TimelineExamples"; From 784c02b1f1527bfda1dd220c2949afadc7f2d295 Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Mon, 16 Sep 2024 19:26:40 -0700 Subject: [PATCH 06/11] frame loader buffering --- .../SchemaIO/components/FrameLoaderView.tsx | 41 +++++++++++++------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx index dafdab5433..657ef42a71 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx @@ -31,24 +31,40 @@ export default function FrameLoaderView(props: ViewPropsType) { React.useState(DEFAULT_FRAME_NUMBER); const triggerEvent = usePanelEvent(); const setPanelState = useSetPanelStateById(true); + const localIdRef = React.useRef(); + + useEffect(() => { + localIdRef.current = Math.random().toString(36).substring(7); + if (data?.frames) + dispatchEvent( + new CustomEvent(`frames-loaded`, { + detail: { localId: localIdRef.current }, + }) + ); + }, [JSON.stringify(data?.frames)]); const loadRange = React.useCallback( async (range: BufferRange) => { - console.log("loadRange", range); if (on_load_range) { - return triggerEvent(panelId, { + triggerEvent(panelId, { params: { range }, operator: on_load_range, }); + return new Promise((resolve) => { + window.addEventListener(`frames-loaded`, (e) => { + if ( + e instanceof CustomEvent && + e.detail.localId === localIdRef.current + ) { + resolve(); + } + }); + }); } }, - [triggerEvent, on_load_range] + [triggerEvent, on_load_range, localIdRef.current] ); - // useEffect(() => { - // loadRange([0, 50]); - // }, []); - const [currentFrame, setCurrentFrame] = useState(DEFAULT_FRAME_NUMBER); const myRenderFrame = React.useCallback( @@ -56,15 +72,11 @@ export default function FrameLoaderView(props: ViewPropsType) { setMyLocalFrameNumber(frameNumber); // console.log("rendering frame", frameNumber, props); setPanelState(panelId, (current) => { - const currentFrameData = data?.frames[frameNumber] || {}; const currentData = current.data ? _.cloneDeep(current.data) : {}; // Clone the object + const currentFrameData = _.get(currentData, path, { frames: [] }) + .frames[frameNumber]; let updatedData = { ...currentData }; - - console.log("data?.frames", data?.frames); - console.log("target", target); _.set(updatedData, target, currentFrameData); // Use lodash set to update safely - console.log("updatedData", updatedData); - return { ...current, data: updatedData }; }); setCurrentFrame(frameNumber); @@ -73,14 +85,17 @@ export default function FrameLoaderView(props: ViewPropsType) { ); const { isTimelineInitialized, subscribe } = useTimeline(); + const [subscribed, setSubscribed] = useState(false); React.useEffect(() => { + if (subscribed) return; if (isTimelineInitialized) { subscribe({ id: `sub1`, loadRange, renderFrame: myRenderFrame, }); + setSubscribed(true); } }, [isTimelineInitialized, loadRange, myRenderFrame, subscribe]); From ec678933e7753c331996b1793841d1994c068c80 Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Fri, 20 Sep 2024 07:59:51 -0700 Subject: [PATCH 07/11] package.json for playback fix --- app/packages/playback/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/packages/playback/package.json b/app/packages/playback/package.json index 2bfbef3378..081b41a9b4 100644 --- a/app/packages/playback/package.json +++ b/app/packages/playback/package.json @@ -1,6 +1,6 @@ { "name": "@fiftyone/playback", - "main": "./src/index.ts", + "main": "./index.ts", "packageManager": "yarn@3.2.1", "devDependencies": { "@eslint/compat": "^1.1.1", From 1d7f0c3f7463a57eeebcf3c69916e6d5de62cbfe Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Mon, 23 Sep 2024 19:34:41 -0700 Subject: [PATCH 08/11] remove old test code --- app/packages/embeddings/src/Testing.tsx | 43 ------------------------- app/packages/embeddings/src/index.ts | 5 --- 2 files changed, 48 deletions(-) delete mode 100644 app/packages/embeddings/src/Testing.tsx diff --git a/app/packages/embeddings/src/Testing.tsx b/app/packages/embeddings/src/Testing.tsx deleted file mode 100644 index da3b9572f3..0000000000 --- a/app/packages/embeddings/src/Testing.tsx +++ /dev/null @@ -1,43 +0,0 @@ -export function main() { - registerComponent({ - name: "TimelineCreator", - label: "TimelineCreator", - component: TimelineCreator, - type: PluginComponentType.Panel, - activator: () => true, - panelOptions: { - surfaces: "modal", - }, - }); -} - -import { BufferRange } from "@fiftyone/utilities"; -import React from "react"; -import { - DEFAULT_FRAME_NUMBER, - GLOBAL_TIMELINE_ID, - SEEK_BAR_DEBOUNCE, -} from "@fiftyone/playback/src/lib/constants"; -import { TimelineName } from "@fiftyone/playback/src/lib/state"; -import { useCreateTimeline } from "@fiftyone/playback/src/lib/use-create-timeline"; -import { useTimeline } from "@fiftyone/playback/src/lib/use-timeline"; -import { useTimelineVizUtils } from "@fiftyone/playback/src/lib/use-timeline-viz-utils"; -import { - FoTimelineContainer, - FoTimelineControlsContainer, - Playhead, - Seekbar, - SeekbarThumb, - Speed, - StatusIndicator, -} from "@fiftyone/playback/src/views/PlaybackElements"; -import { PluginComponentType, registerComponent } from "@fiftyone/plugins"; -import { useDefaultTimelineName } from "@fiftyone/playback/src/lib/use-default-timeline-name"; -import { Timeline } from "@fiftyone/playback/src/views/Timeline"; - -interface TimelineProps { - name?: TimelineName; - style?: React.CSSProperties; -} - -import { TimelineCreator } from "@fiftyone/playback/src/views/TimelineExamples"; diff --git a/app/packages/embeddings/src/index.ts b/app/packages/embeddings/src/index.ts index 86ae1050c9..1e8524ec9a 100644 --- a/app/packages/embeddings/src/index.ts +++ b/app/packages/embeddings/src/index.ts @@ -16,8 +16,3 @@ registerComponent({ priority: BUILT_IN_PANEL_PRIORITY_CONST, }, }); - -// registerOperator(new OpenEmbeddingsPanel()); -import { main } from "./Testing"; - -main(); From 33dccf062c87f2bc411b4b0937a4d59334586f4d Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Tue, 24 Sep 2024 14:34:39 -0700 Subject: [PATCH 09/11] frameloader cleanup --- .../core/src/plugins/SchemaIO/components/FrameLoaderView.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx index 657ef42a71..862d015fb3 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx @@ -91,7 +91,7 @@ export default function FrameLoaderView(props: ViewPropsType) { if (subscribed) return; if (isTimelineInitialized) { subscribe({ - id: `sub1`, + id: timeline_id || GLOBAL_TIMELINE_ID, loadRange, renderFrame: myRenderFrame, }); @@ -99,5 +99,5 @@ export default function FrameLoaderView(props: ViewPropsType) { } }, [isTimelineInitialized, loadRange, myRenderFrame, subscribe]); - return

{currentFrame}

; + return null; } From da6d1da0923a7b21ac2a983cd19a5b8c05902cd4 Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Tue, 24 Sep 2024 16:20:01 -0700 Subject: [PATCH 10/11] framelaoder fixes --- .../SchemaIO/components/FrameLoaderView.tsx | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx index 862d015fb3..5467485755 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx @@ -3,6 +3,7 @@ import React, { useCallback, useEffect, useMemo, + useRef, useState, } from "react"; import { ObjectSchemaType, ViewPropsType } from "../utils/types"; @@ -10,7 +11,7 @@ import { DEFAULT_FRAME_NUMBER, GLOBAL_TIMELINE_ID, } from "@fiftyone/playback/src/lib/constants"; -import { BufferRange } from "@fiftyone/utilities"; +import { BufferManager, BufferRange } from "@fiftyone/utilities"; import { usePanelEvent } from "@fiftyone/operators"; import { usePanelId, @@ -32,30 +33,49 @@ export default function FrameLoaderView(props: ViewPropsType) { const triggerEvent = usePanelEvent(); const setPanelState = useSetPanelStateById(true); const localIdRef = React.useRef(); + const bufm = useRef(new BufferManager()); useEffect(() => { localIdRef.current = Math.random().toString(36).substring(7); + // console.log("localIdRef", localIdRef.current); if (data?.frames) + // console.log("dispatching frames-loaded", localIdRef.current); dispatchEvent( new CustomEvent(`frames-loaded`, { detail: { localId: localIdRef.current }, }) ); - }, [JSON.stringify(data?.frames)]); + }, [data?.signature]); // remove this JSON.strignify const loadRange = React.useCallback( async (range: BufferRange) => { if (on_load_range) { - triggerEvent(panelId, { - params: { range }, - operator: on_load_range, - }); + // if (!bufm.current.containsRange(range)) { + // // only trigger event if the range is not already in the buffer + // await triggerEvent(panelId, { + // params: { range }, + // operator: on_load_range, + // }); + // } + const unp = bufm.current.getUnprocessedBufferRange(range); + const isProcessed = unp === null; + + if (!isProcessed) { + await triggerEvent(panelId, { + params: { range: unp }, + operator: on_load_range, + }); + } + console.log("loading range", range); return new Promise((resolve) => { window.addEventListener(`frames-loaded`, (e) => { + // console.log("frames loaded", e, {'current': localIdRef.current, 'detail': e.detail.localId}); if ( e instanceof CustomEvent && e.detail.localId === localIdRef.current ) { + // console.log("resolving"); + bufm.current.addNewRange(range); resolve(); } }); From 4d9f178636ba50dd76bab810b4536aa7e6c3c99b Mon Sep 17 00:00:00 2001 From: Ritchie Martori Date: Tue, 24 Sep 2024 17:15:47 -0700 Subject: [PATCH 11/11] more frameloader cleanup --- .../SchemaIO/components/FrameLoaderView.tsx | 38 +++---------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx index 5467485755..ff41d1274e 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/FrameLoaderView.tsx @@ -1,11 +1,4 @@ -import React, { - forwardRef, - useCallback, - useEffect, - useMemo, - useRef, - useState, -} from "react"; +import React, { useEffect, useRef, useState } from "react"; import { ObjectSchemaType, ViewPropsType } from "../utils/types"; import { DEFAULT_FRAME_NUMBER, @@ -13,23 +6,15 @@ import { } from "@fiftyone/playback/src/lib/constants"; import { BufferManager, BufferRange } from "@fiftyone/utilities"; import { usePanelEvent } from "@fiftyone/operators"; -import { - usePanelId, - usePanelState, - useSetPanelStateById, -} from "@fiftyone/spaces"; +import { usePanelId, useSetPanelStateById } from "@fiftyone/spaces"; import { useTimeline } from "@fiftyone/playback/src/lib/use-timeline"; import _ from "lodash"; -import { useDefaultTimelineName } from "@fiftyone/playback/src/lib/use-default-timeline-name"; export default function FrameLoaderView(props: ViewPropsType) { const { schema, path, data } = props; const { view = {} } = schema; const { on_load_range, timeline_id, target } = view; - const { properties } = schema as ObjectSchemaType; const panelId = usePanelId(); - const [myLocalFrameNumber, setMyLocalFrameNumber] = - React.useState(DEFAULT_FRAME_NUMBER); const triggerEvent = usePanelEvent(); const setPanelState = useSetPanelStateById(true); const localIdRef = React.useRef(); @@ -37,26 +22,17 @@ export default function FrameLoaderView(props: ViewPropsType) { useEffect(() => { localIdRef.current = Math.random().toString(36).substring(7); - // console.log("localIdRef", localIdRef.current); if (data?.frames) - // console.log("dispatching frames-loaded", localIdRef.current); - dispatchEvent( + window.dispatchEvent( new CustomEvent(`frames-loaded`, { detail: { localId: localIdRef.current }, }) ); - }, [data?.signature]); // remove this JSON.strignify + }, [data?.signature]); const loadRange = React.useCallback( async (range: BufferRange) => { if (on_load_range) { - // if (!bufm.current.containsRange(range)) { - // // only trigger event if the range is not already in the buffer - // await triggerEvent(panelId, { - // params: { range }, - // operator: on_load_range, - // }); - // } const unp = bufm.current.getUnprocessedBufferRange(range); const isProcessed = unp === null; @@ -66,15 +42,13 @@ export default function FrameLoaderView(props: ViewPropsType) { operator: on_load_range, }); } - console.log("loading range", range); + return new Promise((resolve) => { window.addEventListener(`frames-loaded`, (e) => { - // console.log("frames loaded", e, {'current': localIdRef.current, 'detail': e.detail.localId}); if ( e instanceof CustomEvent && e.detail.localId === localIdRef.current ) { - // console.log("resolving"); bufm.current.addNewRange(range); resolve(); } @@ -89,8 +63,6 @@ export default function FrameLoaderView(props: ViewPropsType) { const myRenderFrame = React.useCallback( (frameNumber: number) => { - setMyLocalFrameNumber(frameNumber); - // console.log("rendering frame", frameNumber, props); setPanelState(panelId, (current) => { const currentData = current.data ? _.cloneDeep(current.data) : {}; // Clone the object const currentFrameData = _.get(currentData, path, { frames: [] })