Skip to content

Commit

Permalink
[civitai] update default archive format (#6302)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mikf
mikf committed Oct 10, 2024
1 parent 29e46f6 commit bc954eb
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 94 deletions.
2 changes: 1 addition & 1 deletion gallery_dl/extractor/civitai.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class CivitaiExtractor(Extractor):
root = "https://civitai.com"
directory_fmt = ("{category}", "{username|user[username]}", "images")
filename_fmt = "{file[id]|id|filename}.{extension}"
archive_fmt = "{hash}"
archive_fmt = "{file[hash]|hash}"
request_interval = (0.5, 1.5)

def _init(self):
Expand Down
176 changes: 83 additions & 93 deletions test/results/civitai.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,29 +12,20 @@
"#url" : "https://civitai.com/models/703211/maid-classic",
"#class": civitai.CivitaiModelExtractor,
"#urls" : [
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/5c4efa68-bb58-47c5-a716-98cd0f51f047/original=true/26962950.jpeg",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/original=true/26962948.jpeg",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/2dd1dc69-45a6-4beb-b36b-2e2bc65e3cda/original=true/26962957.jpeg",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/52b6efa7-801c-4901-90b4-fa3964d23480/original=true/26887862.jpeg",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c/original=true/26887856.jpeg",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/68568d22-c4f3-45cb-ac32-82f1cedf968f/original=true/26887852.jpeg",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/5c4efa68-bb58-47c5-a716-98cd0f51f047/original=true/00013-4238863814.png",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/original=true/00014-3150861441.png",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/2dd1dc69-45a6-4beb-b36b-2e2bc65e3cda/original=true/00015-2885514572.png",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/52b6efa7-801c-4901-90b4-fa3964d23480/original=true/00004-822988489.png",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c/original=true/00005-1059918744.png",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/68568d22-c4f3-45cb-ac32-82f1cedf968f/original=true/00006-3467286319.png",
],

"model" : {
"allowCommercialUse": ["RentCivit"],
"allowDerivatives": True,
"allowDifferentLicense": True,
"allowNoCredit": True,
"cosmetic" : None,
"description": "<p>The strength of Lora is recommended to be around 1.0.</p>",
"id" : 703211,
"minor" : False,
"name" : "メイド　クラシック/maid classic",
"nsfw" : False,
"nsfwLevel" : 1,
"poi" : False,
"stats" : dict,
"tags" : ["clothing"],
"type" : "LORA"
},
"user" : {
Expand All @@ -47,35 +38,22 @@
},

{
"#url" : "https://civitai.com/models/703211?modelVersionId=786644",
"#class": civitai.CivitaiModelExtractor,
"#urls" : [
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/52b6efa7-801c-4901-90b4-fa3964d23480/original=true/26887862.jpeg",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c/original=true/26887856.jpeg",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/68568d22-c4f3-45cb-ac32-82f1cedf968f/original=true/26887852.jpeg",
"#url" : "https://civitai.com/models/703211?modelVersionId=786644",
"#comment": "model version ID",
"#class" : civitai.CivitaiModelExtractor,
"#urls" : [
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/52b6efa7-801c-4901-90b4-fa3964d23480/original=true/00004-822988489.png",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c/original=true/00005-1059918744.png",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/68568d22-c4f3-45cb-ac32-82f1cedf968f/original=true/00006-3467286319.png",
],

"version": {
"availability": "Public",
"baseModel" : "Pony",
"createdAt" : "2024-08-30T15:28:47.661Z",
"date" : "dt:2024-08-30 15:28:47",
"downloadUrl" : "https://civitai.com/api/download/models/786644",
"files" : list,
"id" : 786644,
"images" : list,
"index" : 1,
"name" : "v1.0 pony",
"nsfwLevel" : 1,
"publishedAt" : "2024-08-30T15:39:17.674Z",
"stats" : dict,
"status" : "Published",
"trainedWords": [
"maid",
"madi apron",
"maid headdress",
"long sleeves",
],
},
"user" : {
"image" : None,
Expand All @@ -94,74 +72,86 @@
"#url" : "https://civitai.com/images/26962948",
"#class": civitai.CivitaiImageExtractor,
"#options" : {"quality": "w"},
"#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/w/69bf3279-df2c-4ec8-b795-479e9cd3db1b.jpeg",
"#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/w/00014-3150861441.png",
"#sha1_content": "a9a9d08f5fcdbc1e1eec7f203717f9df97b7a671",

"browsingLevel": 1,
"createdAt": "2024-08-31T01:11:47.021Z",
"date" : "dt:2024-08-31 01:11:47",
"extension": "jpg",
"filename" : "69bf3279-df2c-4ec8-b795-479e9cd3db1b",
"filename" : "00014-3150861441",
"hash" : "ULN0-w?b4nRjxGM{-;t7M_t7NGae~qRjMyt7",
"height" : 1536,
"id" : 26962948,
"meta": {
"Denoising strength": "0.4",
"Model": "boleromix_XL_V1.3",
"Model hash": "afaf521da2",
"Size": "1152x1536",
"TI hashes": {
"negativeXL_D": "fff5d51ab655"
},
"Tiled Diffusion scale factor": "1.5",
"Tiled Diffusion upscaler": "R-ESRGAN 4x+ Anime6B",
"VAE": "sdxl_vae.safetensors",
"Version": "v1.7.0",
"cfgScale": 7,
"hashes": {
"lora:add-detail-xl": "9c783c8ce46c",
"lora:classic maid_XL_V1.0": "e8f6e4297112",
"model": "afaf521da2",
"vae": "735e4c3a44",
},
"negativePrompt": "negativeXL_D,(worst quality,extra legs,extra arms,extra ears,bad fingers,extra fingers,bad anatomy, missing fingers, lowres,username, artist name, text,pubic hair,bar censor,censored,multipul angle,split view,realistic,3D:1)",
"prompt": "masterpiece,ultra-detailed,best quality,8K,illustration,cute face,clean skin ,shiny hair,girl,ultra-detailed-eyes,simple background, <lora:add-detail-xl:1> <lora:classic maid_XL_V1.0:1> maid, maid apron, maid headdress, long sleeves,tray,tea,cup,skirt lift",
"resources": [
{
"hash": "9c783c8ce46c",
"name": "add-detail-xl",
"type": "lora",
"weight": 1,
},
{
"hash": "e8f6e4297112",
"name": "classic maid_XL_V1.0",
"type": "lora",
},
{
"hash": "afaf521da2",
"name": "boleromix_XL_V1.3",
"type": "model",
},
],
"sampler": "DPM++ 2M Karras",
"seed": 3150861441,
"steps": 20,
"nsfwLevel": 1,
"postId" : 6030721,
"stats" : dict,
"url" : "69bf3279-df2c-4ec8-b795-479e9cd3db1b",
"user" : {
"username": "bolero537",
},
"nsfw": False,
"nsfwLevel": "None",
"postId": 6030721,
"stats": {
"commentCount": int,
"cryCount" : int,
"dislikeCount": int,
"heartCount" : int,
"laughCount" : int,
"likeCount" : int,
"width" : 1152,
},

{
"#url" : "https://civitai.com/posts/6877551",
"#class": civitai.CivitaiPostExtractor,
"#urls" : [
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6220fa0f-9037-4b1d-bfbd-a740a06eeb7c/original=true/30748752.png",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/cd1edb7f-7b50-4da5-bf23-d38f24d8aef0/original=true/30748747.png",
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/cfd5b231-accd-49bd-8bde-370880f63aa6/original=true/30748733.png",
],

"post": {
"id" : 6877551,
"date": "dt:2024-09-22 12:54:15",
},
"url": "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/width=1152/69bf3279-df2c-4ec8-b795-479e9cd3db1b.jpeg",
"username": "bolero537",
"width": 1152,
},

{
"#url" : "https://civitai.com/tag/mecha",
"#class": civitai.CivitaiTagModelsExtractor,
},

{
"#url" : "https://civitai.com/images?tags=482",
"#class": civitai.CivitaiTagImagesExtractor,
},

{
"#url" : "https://civitai.com/search/models?sortBy=models_v9&query=mecha",
"#class": civitai.CivitaiSearchExtractor,
},

{
"#url" : "https://civitai.com/user/waomodder",
"#class": civitai.CivitaiUserExtractor,
"#urls" : [
"https://civitai.com/user/waomodder/models",
"https://civitai.com/user/waomodder/posts",
],
},

{
"#url" : "https://civitai.com/user/waomodder/models",
"#class": civitai.CivitaiUserModelsExtractor,
"#pattern": civitai.CivitaiModelExtractor.pattern,
"#count" : ">= 8",
},

{
"#url" : "https://civitai.com/user/waomodder/posts",
"#class": civitai.CivitaiUserPostsExtractor,
"#pattern": r"https://image\.civitai\.com/xG1nkqKTMzGDvpLrqFT7WA/[0-9a-f-]+/original=true/\S+\.(jpe?g|png)",
"#range" : "1-50",
"#count" : 50,
},

{
"#url" : "https://civitai.com/user/waomodder/images",
"#class": civitai.CivitaiUserImagesExtractor,
"#pattern": r"https://image\.civitai\.com/xG1nkqKTMzGDvpLrqFT7WA/[0-9a-f-]+/original=true/\S+\.png",
"#range" : "1-50",
"#count" : 50,
},

)

0 comments on commit bc954eb

Please sign in to comment.