From f475d875db8e73eb7399a57f9b4620d0cb3090b5 Mon Sep 17 00:00:00 2001 From: ahaapple Date: Tue, 10 Sep 2024 19:09:32 +0800 Subject: [PATCH] Show videos in search result --- .../(search)/search/[id]/search-result.tsx | 9 +--- .../components/search/expandable-section.tsx | 38 ++++++++++++++ .../search/search-message-bubble.tsx | 31 +++++------ frontend/components/search/search-window.tsx | 17 ++++-- frontend/components/search/video-gallery.tsx | 28 ++++++++++ .../components/sidebar/sidebar-actions.tsx | 52 ++++--------------- frontend/lib/search/search.ts | 3 +- frontend/lib/search/serper.ts | 28 ++++++++-- frontend/lib/server-utils.ts | 15 ++++-- frontend/lib/store/search.ts | 31 ++++------- frontend/lib/tools/auto.ts | 21 ++++++-- frontend/lib/tools/indie.ts | 2 +- frontend/lib/tools/knowledge-base.ts | 2 +- frontend/lib/types.ts | 6 +++ 14 files changed, 175 insertions(+), 108 deletions(-) create mode 100644 frontend/components/search/expandable-section.tsx create mode 100644 frontend/components/search/video-gallery.tsx diff --git a/frontend/app/[locale]/(search)/search/[id]/search-result.tsx b/frontend/app/[locale]/(search)/search/[id]/search-result.tsx index 1f17879..3013317 100644 --- a/frontend/app/[locale]/(search)/search/[id]/search-result.tsx +++ b/frontend/app/[locale]/(search)/search/[id]/search-result.tsx @@ -17,7 +17,6 @@ export default function SearchResult({ id, user }: SearchPageProps) { useEffect(() => { if (!search) { - console.error('fetching search:', id, searches.length); const fetchSearch = async () => { const search = await getSearch(id, user.id); if (search) { @@ -28,11 +27,5 @@ export default function SearchResult({ id, user }: SearchPageProps) { } }, [id, user, search]); - return ( - - ); + return ; } diff --git a/frontend/components/search/expandable-section.tsx b/frontend/components/search/expandable-section.tsx new file mode 100644 index 0000000..c0e3191 --- /dev/null +++ b/frontend/components/search/expandable-section.tsx @@ -0,0 +1,38 @@ +'use client'; + +import { Minus, Plus } from 'lucide-react'; +import React, { useEffect, useState } from 'react'; + +const ExpandableSection = ({ title, icon: Icon, children }) => { + const [isOpen, setIsOpen] = useState(true); + + useEffect(() => { + const checkScreenSize = () => setIsOpen(window.innerWidth >= 768); + checkScreenSize(); + window.addEventListener('resize', checkScreenSize); + return () => window.removeEventListener('resize', checkScreenSize); + }, []); + + return ( +
+
+ { + e.preventDefault(); + setIsOpen(!isOpen); + }} + > +
+ +

{title}

+
+
{isOpen ? : }
+ + {isOpen && children} +
+
+ ); +}; + +export default ExpandableSection; diff --git a/frontend/components/search/search-message-bubble.tsx b/frontend/components/search/search-message-bubble.tsx index 57395c8..2a47353 100644 --- a/frontend/components/search/search-message-bubble.tsx +++ b/frontend/components/search/search-message-bubble.tsx @@ -1,5 +1,5 @@ import SourceBubble from '@/components/search/source-bubble'; -import { FileTextIcon, Images, ListPlusIcon, PlusIcon, TextSearchIcon } from 'lucide-react'; +import { FileTextIcon, Film, Images, ListPlusIcon, PlusIcon, TextSearchIcon, Youtube } from 'lucide-react'; import ImageGallery from '@/components/search/image-gallery'; import { Message } from '@/lib/types'; @@ -8,6 +8,8 @@ import AnswerSection from '@/components/search/answer-section'; import QuestionSection from '@/components/search/question-section'; import ActionButtons from '@/components/search/action-buttons'; import { extractAllImageUrls } from '@/lib/shared-utils'; +import VideoGallery from '@/components/search/video-gallery'; +import ExpandableSection from '@/components/search/expandable-section'; const SearchMessageBubble = memo( (props: { searchId: string; message: Message; onSelect: (question: string) => void; reload: (msgId: string) => void; isLoading: boolean }) => { @@ -20,6 +22,7 @@ const SearchMessageBubble = memo( const message = props.message; const sources = message.sources ?? []; const images = message.images ?? []; + const videos = message.videos ?? []; const searchId = props.searchId; const attachments = useMemo(() => { @@ -37,30 +40,28 @@ const SearchMessageBubble = memo(
{!isUser && content && } {(images.length > 0 || !isLoading) && !isUser && } - {!isUser && sources.length > 0 && ( -
-
- -

Sources

-
-
+ {sources.length > 0 && ( + +
{sources.map((source, index) => (
))}
-
+ )} {images.length > 0 && ( -
-
- -

Images

-
+ -
+ + )} + + {videos.length > 0 && ( + + + )} {related && ( diff --git a/frontend/components/search/search-window.tsx b/frontend/components/search/search-window.tsx index b332fb6..71d31cd 100644 --- a/frontend/components/search/search-window.tsx +++ b/frontend/components/search/search-window.tsx @@ -9,7 +9,7 @@ import { useSigninModal } from '@/hooks/use-signin-modal'; import SearchBar from '@/components/search-bar'; import { configStore, useProfileStore } from '@/lib/store'; -import { ImageSource, Message, TextSource, User } from '@/lib/types'; +import { ImageSource, Message, TextSource, User, VideoSource } from '@/lib/types'; import { generateId } from 'ai'; import { LoaderCircle } from 'lucide-react'; import { useScrollAnchor } from '@/hooks/use-scroll-anchor'; @@ -99,7 +99,13 @@ export function SearchWindow({ id, initialMessages, user, isReadOnly = false }: let accumulatedRelated = ''; let messageIndex: number | null = null; - const updateMessages = (parsedResult?: string, newSources?: TextSource[], newImages?: ImageSource[], newRelated?: string) => { + const updateMessages = ( + parsedResult?: string, + newSources?: TextSource[], + newImages?: ImageSource[], + newRelated?: string, + newVideos?: VideoSource[], + ) => { const activeSearch = useSearchStore.getState().activeSearch; if (messageIndex === null || !activeSearch.messages[messageIndex]) { messageIndex = activeSearch.messages.length; @@ -112,6 +118,7 @@ export function SearchWindow({ id, initialMessages, user, isReadOnly = false }: sources: newSources || [], images: newImages || [], related: newRelated || '', + videos: newVideos || [], role: 'assistant', }, ], @@ -127,6 +134,7 @@ export function SearchWindow({ id, initialMessages, user, isReadOnly = false }: content: parsedResult ? parsedResult.trim() : msg.content, sources: newSources || msg.sources, images: newImages || msg.images, + videos: newVideos || msg.videos, related: newRelated || msg.related, }; } @@ -200,7 +208,7 @@ export function SearchWindow({ id, initialMessages, user, isReadOnly = false }: setIsLoading(false); }, onmessage(msg) { - const { clear, answer, status, sources, images, related } = JSON.parse(msg.data); + const { clear, answer, status, sources, images, related, videos } = JSON.parse(msg.data); if (clear) { accumulatedMessage = ''; updateMessages(accumulatedMessage); @@ -213,6 +221,7 @@ export function SearchWindow({ id, initialMessages, user, isReadOnly = false }: sources, images, related ? (accumulatedRelated += related) : undefined, + videos, ); }, }); @@ -222,7 +231,7 @@ export function SearchWindow({ id, initialMessages, user, isReadOnly = false }: toast.error('An error occurred while searching, please refresh your page and try again'); } }, - [input, isReadOnly, isLoading, signInModal, addSearch, updateActiveSearch, user?.id], + [input, isReadOnly, isLoading, signInModal, addSearch, updateActiveSearch, upgradeModal, user], ); const sendSelectedQuestion = useCallback( diff --git a/frontend/components/search/video-gallery.tsx b/frontend/components/search/video-gallery.tsx new file mode 100644 index 0000000..14e01ea --- /dev/null +++ b/frontend/components/search/video-gallery.tsx @@ -0,0 +1,28 @@ +import { VideoSource } from '@/lib/types'; +import React, { memo } from 'react'; + +type VideoGalleryProps = { + videos: VideoSource[]; +}; + +const VideoGallery: React.FC = memo(({ videos }) => { + return ( +
+ {videos.map((video, index) => ( +
+ +
+ ))} +
+ ); +}); + +VideoGallery.displayName = 'VideoGallery'; +export default VideoGallery; diff --git a/frontend/components/sidebar/sidebar-actions.tsx b/frontend/components/sidebar/sidebar-actions.tsx index d24229d..261ff45 100644 --- a/frontend/components/sidebar/sidebar-actions.tsx +++ b/frontend/components/sidebar/sidebar-actions.tsx @@ -16,27 +16,17 @@ import { AlertDialogTitle, } from '@/components/ui/alert-dialog'; import { Button } from '@/components/ui/button'; -import { - Tooltip, - TooltipContent, - TooltipTrigger, -} from '@/components/ui/tooltip'; +import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'; import { LoaderCircle, Share2, Trash2 } from 'lucide-react'; import { SearchShareDialog } from '@/components/search/search-share-dialog'; import { useSearchStore } from '@/lib/store/local-history'; interface SidebarActionsProps { search: Search; - removeSearch: (args: { - id: string; - path: string; - }) => ServerActionResult; + removeSearch: (args: { id: string; path: string }) => ServerActionResult; } -export function SidebarActions({ - search: search, - removeSearch: removeSearch, -}: SidebarActionsProps) { +export function SidebarActions({ search: search, removeSearch: removeSearch }: SidebarActionsProps) { const router = useRouter(); const [deleteDialogOpen, setDeleteDialogOpen] = React.useState(false); const [shareDialogOpen, setShareDialogOpen] = React.useState(false); @@ -47,11 +37,7 @@ export function SidebarActions({
- @@ -73,30 +59,15 @@ export function SidebarActions({ Delete It
- setShareDialogOpen(false)} - /> - + setShareDialogOpen(false)} /> + - - Are you absolutely sure? - - - This will permanently delete your search message and - remove your data from our servers. - + Are you absolutely sure? + This will permanently delete your search message and remove your data from our servers. - - Cancel - + Cancel { @@ -115,15 +86,12 @@ export function SidebarActions({ } setDeleteDialogOpen(false); - // router.refresh(); router.push('/'); toast.success('Search deleted'); }); }} > - {isRemovePending && ( - - )} + {isRemovePending && } Delete diff --git a/frontend/lib/search/search.ts b/frontend/lib/search/search.ts index 015d8e4..3641240 100644 --- a/frontend/lib/search/search.ts +++ b/frontend/lib/search/search.ts @@ -2,7 +2,7 @@ import 'server-only'; import { SerperSearch } from '@/lib/search/serper'; import { VectorSearch } from '@/lib/search/vector'; -import { ImageSource, SearchCategory, TextSource } from '@/lib/types'; +import { ImageSource, SearchCategory, TextSource, VideoSource } from '@/lib/types'; import { EXASearch } from '@/lib/search/exa'; export interface SearchOptions { @@ -22,6 +22,7 @@ export interface AnySource { export interface SearchResult { texts: TextSource[]; images: ImageSource[]; + videos?: VideoSource[]; } export interface SearchSource { diff --git a/frontend/lib/search/serper.ts b/frontend/lib/search/serper.ts index 8864d3c..83a0bb8 100644 --- a/frontend/lib/search/serper.ts +++ b/frontend/lib/search/serper.ts @@ -1,9 +1,9 @@ import 'server-only'; import { SearchOptions, SearchResult, SearchSource } from '@/lib/search/search'; -import { ImageSource, SearchCategory, TextSource } from '@/lib/types'; +import { ImageSource, SearchCategory, TextSource, VideoSource } from '@/lib/types'; import { logError } from '@/lib/log'; -import { fetchWithTimeout } from '@/lib/server-utils'; +import { extractYouTubeId, fetchWithTimeout } from '@/lib/server-utils'; import { SERPER_API_KEY } from '@/lib/env'; const serperUrl = 'https://google.serper.dev/'; @@ -16,6 +16,8 @@ function formatUrl(options: SearchOptions) { return `${serperUrl}images`; case SearchCategory.NEWS: return `${serperUrl}news`; + case SearchCategory.VIDEOS: + return `${serperUrl}videos`; default: return `${serperUrl}search`; } @@ -39,6 +41,7 @@ export class SerperSearch implements SearchSource { let texts: TextSource[] = []; let images: ImageSource[] = []; + let videos: VideoSource[] = []; let jsonResponse; try { const response = await fetchWithTimeout(url, { @@ -90,6 +93,23 @@ export class SerperSearch implements SearchSource { ); } + if (jsonResponse.videos) { + videos = videos.concat( + jsonResponse.videos + .map((v: any) => { + const videoId = extractYouTubeId(v.link); + if (!videoId) { + return null; + } + return { + title: v.title, + id: videoId, + }; + }) + .filter((video): video is NonNullable => video !== null), + ); + } + if (jsonResponse.organic) { texts = texts.concat( jsonResponse.organic.map((c: any) => ({ @@ -109,10 +129,10 @@ export class SerperSearch implements SearchSource { })), ); } - return { texts, images }; + return { texts, images, videos }; } catch (error) { logError(error, 'search-serper'); - return { texts, images }; + return { texts, images, videos }; } } } diff --git a/frontend/lib/server-utils.ts b/frontend/lib/server-utils.ts index 46a0cb0..08a18d9 100644 --- a/frontend/lib/server-utils.ts +++ b/frontend/lib/server-utils.ts @@ -1,17 +1,14 @@ import 'server-only'; import { generateId } from 'ai'; -import { ImageSource, Message as StoreMessage, TextSource } from '@/lib/types'; +import { ImageSource, Message as StoreMessage, TextSource, VideoSource } from '@/lib/types'; import { saveSearch } from '@/lib/store/search'; interface FetchWithTimeoutOptions extends RequestInit { timeout?: number; } -export const fetchWithTimeout = async ( - resource: RequestInfo, - options: FetchWithTimeoutOptions = {}, -): Promise => { +export const fetchWithTimeout = async (resource: RequestInfo, options: FetchWithTimeoutOptions = {}): Promise => { const { timeout = 10000 } = options; const controller = new AbortController(); @@ -27,6 +24,12 @@ export const fetchWithTimeout = async ( } }; +export function extractYouTubeId(url: string): string { + const regExp = /^.*(youtu.be\/|v\/|u\/\w\/|embed\/|watch\?v=|\&v=)([^#\&\?]*).*/; + const match = url.match(regExp); + return match && match[2].length === 11 ? match[2] : ''; +} + export function containsValidUrl(text: string) { const urlPattern = /https?:\/\/[^\s/$.?#].[^\s]*/i; return urlPattern.test(text); @@ -38,6 +41,7 @@ export async function saveMessages( answer: string, texts?: TextSource[], images?: ImageSource[], + videos?: VideoSource[], related?: string, ) { if (!userId) { @@ -50,6 +54,7 @@ export async function saveMessages( content: answer, sources: texts, images: images, + videos: videos, related: related, }); diff --git a/frontend/lib/store/search.ts b/frontend/lib/store/search.ts index fab30c7..8358700 100644 --- a/frontend/lib/store/search.ts +++ b/frontend/lib/store/search.ts @@ -15,23 +15,14 @@ const redis = new Redis({ const SEARCH_KEY = 'search:'; const USER_SEARCH_KEY = 'user:search:'; -export async function getSearches( - userId: string, - offset: number = 0, - limit: number = 20, -) { +export async function getSearches(userId: string, offset: number = 0, limit: number = 20) { console.log('getSearches all userId', userId, offset, limit); try { const pipeline = redis.pipeline(); - const searches: string[] = await redis.zrange( - USER_SEARCH_KEY + userId, - offset, - offset + limit - 1, - { - rev: true, - }, - ); + const searches: string[] = await redis.zrange(USER_SEARCH_KEY + userId, offset, offset + limit - 1, { + rev: true, + }); if (searches.length === 0) { return []; @@ -57,11 +48,7 @@ export async function clearSearches() { } const userId = session.user.id; - const searches: string[] = await redis.zrange( - USER_SEARCH_KEY + userId, - 0, - -1, - ); + const searches: string[] = await redis.zrange(USER_SEARCH_KEY + userId, 0, -1); if (!searches.length) { return; } @@ -111,7 +98,7 @@ export async function removeSearch({ id, path }: { id: string; path: string }) { const session = await auth(); if (!session) { return { - error: 'Unauthorized', + error: 'Unauthorized, please retry', }; } @@ -119,7 +106,7 @@ export async function removeSearch({ id, path }: { id: string; path: string }) { const uid = String(await redis.hget(SEARCH_KEY + id, 'userId')); if (uid !== session?.user?.id) { return { - error: 'Unauthorized', + error: 'Unauthorized, you cannot remove this search', }; } await redis.del(SEARCH_KEY + id); @@ -196,10 +183,10 @@ export async function getSearch(id: string, userId: string) { console.log('getSearch userId', userId, id); try { const search = await redis.hgetall(SEARCH_KEY + id); - if (!search) { + if (!search || search.userId !== userId) { console.warn('getSearch, No search found:', id, userId); + return null; } - // console.log('getSearch userId', userId, id, search); return search; } catch (error) { console.error('Failed to get search:', error, id, userId); diff --git a/frontend/lib/tools/auto.ts b/frontend/lib/tools/auto.ts index 08cfdea..9f4092e 100644 --- a/frontend/lib/tools/auto.ts +++ b/frontend/lib/tools/auto.ts @@ -14,7 +14,7 @@ import { directlyAnswer } from '@/lib/tools/answer'; import { getTopStories } from '@/lib/tools/hacker-news'; import { getRelatedQuestions } from '@/lib/tools/related'; import { searchRelevantContent } from '@/lib/tools/search'; -import { ImageSource, Message as StoreMessage, SearchCategory, TextSource } from '@/lib/types'; +import { ImageSource, Message as StoreMessage, SearchCategory, TextSource, VideoSource } from '@/lib/types'; import { CoreUserMessage, ImagePart, streamText, TextPart, tool } from 'ai'; import util from 'util'; import { z } from 'zod'; @@ -35,6 +35,7 @@ export async function autoAnswer( let texts: TextSource[] = []; let images: ImageSource[] = []; + let videos: VideoSource[] = []; let history = getHistory(isPro, messages); const system = util.format(AutoAnswerPrompt, profile, history); @@ -135,6 +136,10 @@ export async function autoAnswer( .search(query) .then((results) => results.images.filter((img) => img.image.startsWith('https'))); + const videoFetchPromise = getSearchEngine({ + categories: [SearchCategory.VIDEOS], + }).search(query); + fullAnswer = ''; await streamResponse({ status: 'Answering ...', clear: true }, onStream); await directlyAnswer(isPro, source, history, profile, getLLM(model), rewriteQuery, texts, (msg) => { @@ -142,13 +147,19 @@ export async function autoAnswer( onStream?.(JSON.stringify({ answer: msg })); }); - const fetchedImages = await imageFetchPromise; - images = [...images, ...fetchedImages]; - await streamResponse({ images: images, status: 'Generating related questions ...' }, onStream); + await streamResponse({ status: 'Generating related questions ...' }, onStream); await getRelatedQuestions(query, texts, (msg) => { fullRelated += msg; onStream?.(JSON.stringify({ related: msg })); }); + + const fetchedImages = await imageFetchPromise; + images = [...images, ...fetchedImages]; + await streamResponse({ images: images, status: 'Fetch related videos ...' }, onStream); + + const fetchedVideos = await videoFetchPromise; + videos = fetchedVideos.videos; + await streamResponse({ videos: videos }, onStream); } else { await streamResponse({ status: 'Generating related questions ...' }, onStream); await getRelatedQuestions(query, texts, (msg) => { @@ -161,7 +172,7 @@ export async function autoAnswer( console.error(`Failed to increment search count for user ${userId}:`, error); }); - await saveMessages(userId, messages, fullAnswer, texts, images, fullRelated); + await saveMessages(userId, messages, fullAnswer, texts, images, videos, fullRelated); onStream?.(null, true); } catch (error) { logError(error, 'llm-openai'); diff --git a/frontend/lib/tools/indie.ts b/frontend/lib/tools/indie.ts index 5b1faf2..9033484 100644 --- a/frontend/lib/tools/indie.ts +++ b/frontend/lib/tools/indie.ts @@ -89,7 +89,7 @@ export async function indieMakerSearch(messages: StoreMessage[], isPro: boolean, console.error(`Failed to increment search count for user ${userId}:`, error); }); - await saveMessages(userId, messages, fullAnswer, texts, images, fullRelated); + await saveMessages(userId, messages, fullAnswer, texts, images, [], fullRelated); onStream?.(null, true); } catch (error) { logError(error, 'indie-search'); diff --git a/frontend/lib/tools/knowledge-base.ts b/frontend/lib/tools/knowledge-base.ts index 663ae59..ad3876a 100644 --- a/frontend/lib/tools/knowledge-base.ts +++ b/frontend/lib/tools/knowledge-base.ts @@ -52,7 +52,7 @@ export async function knowledgeBaseSearch(messages: StoreMessage[], isPro: boole console.error(`Failed to increment search count for user ${userId}:`, error); }); - await saveMessages(userId, messages, fullAnswer, texts, [], ''); + await saveMessages(userId, messages, fullAnswer, texts, [], [], ''); onStream?.(null, true); } catch (error) { logError(error, 'knowledge-base-search'); diff --git a/frontend/lib/types.ts b/frontend/lib/types.ts index 6a2bb20..27ca973 100644 --- a/frontend/lib/types.ts +++ b/frontend/lib/types.ts @@ -30,6 +30,11 @@ export interface ImageSource { type?: string; } +export interface VideoSource { + title: string; + id: string; +} + export type ServerActionResult = Promise< | Result | { @@ -67,6 +72,7 @@ export type Message = { attachments?: string[]; sources?: TextSource[]; images?: ImageSource[]; + videos?: VideoSource[]; related?: string; };