itman
/
scan-vid
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

записывает в БД

Browse Source
main
Анатолий Тухтаров 11 months ago
parent 0910c8df70
commit 186e468bb0
4 changed files with 130 additions and 197 deletions
Show Stats Download Patch File Download Diff File

26
README.md
Unescape Escape View File

@ -10,5 +10,31 @@
Выводит размер файла - для проверки на маленькие файлы. Выводит размер файла - для проверки на маленькие файлы.
Текущий запрос:
Dict{String, Any}(
"host_id" => ?
"filename" => "d:\\vids\\Ретро, винтаж, классика, порно, секс, эротика, лесби, минет,.mp4",
"newname" => ?,
"size" => 432465677,
"hsize" => "412.43 MB",
"duration" => DateTime("1970-01-01T01:26:39.987"),
"hduration" => "01:26:39",
"width" => 640,
"height" => 360,
"bit_rate" => "24.0",
"ext" => ".mp4",
"crc" => 760779202,
"ctime" => 1.7045445334502406e9,
"hctime" => "06.01.2024",
"mtime" => 1.7045443156928878e9,
"hmtime" => "06.01.2024",
"codec" => "h264",
"poster" => ?,
"audio" => ?,
"tags" => ?,
"type" => ?,
)

177
ffprobe.info
Unescape Escape View File

@ -1,177 +0,0 @@
{
"streams": [
{
"index": 0,
"codec_name": "h264",
"codec_long_name": "H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10",
"profile": "High",
"codec_type": "video",
"codec_tag_string": "avc1",
"codec_tag": "0x31637661",
"width": 1024,
"height": 512,
"coded_width": 1024,
"coded_height": 512,
"closed_captions": 0,
"film_grain": 0,
"has_b_frames": 2,
"sample_aspect_ratio": "1:1",
"display_aspect_ratio": "2:1",
"pix_fmt": "yuv420p",
"level": 31,
"chroma_location": "left",
"field_order": "progressive",
"refs": 1,
"is_avc": "true",
"nal_length_size": "4",
"id": "0x1",
"r_frame_rate": "24000/1001",
"avg_frame_rate": "24000/1001",
"time_base": "1/24000",
"start_pts": 0,
"start_time": "0.000000",
"duration_ts": 8857849,
"duration": "369.077042",
"bit_rate": "855528",
"bits_per_raw_sample": "8",
"nb_frames": "8849",
"extradata_size": 41,
"disposition": {
"default": 1,
"dub": 0,
"original": 0,
"comment": 0,
"lyrics": 0,
"karaoke": 0,
"forced": 0,
"hearing_impaired": 0,
"visual_impaired": 0,
"clean_effects": 0,
"attached_pic": 0,
"timed_thumbnails": 0,
"captions": 0,
"descriptions": 0,
"metadata": 0,
"dependent": 0,
"still_image": 0
},
"tags": {
"language": "eng",
"handler_name": "VideoHandler",
"vendor_id": "[0][0][0][0]"
}
},
{
"index": 1,
"codec_name": "aac",
"codec_long_name": "AAC (Advanced Audio Coding)",
"profile": "LC",
"codec_type": "audio",
"codec_tag_string": "mp4a",
"codec_tag": "0x6134706d",
"sample_fmt": "fltp",
"sample_rate": "48000",
"channels": 2,
"channel_layout": "stereo",
"bits_per_sample": 0,
"id": "0x2",
"r_frame_rate": "0/0",
"avg_frame_rate": "0/0",
"time_base": "1/48000",
"start_pts": 0,
"start_time": "0.000000",
"duration_ts": 17713502,
"duration": "369.031292",
"bit_rate": "128038",
"nb_frames": "17299",
"extradata_size": 2,
"disposition": {
"default": 1,
"dub": 0,
"original": 0,
"comment": 0,
"lyrics": 0,
"karaoke": 0,
"forced": 0,
"hearing_impaired": 0,
"visual_impaired": 0,
"clean_effects": 0,
"attached_pic": 0,
"timed_thumbnails": 0,
"captions": 0,
"descriptions": 0,
"metadata": 0,
"dependent": 0,
"still_image": 0
},
"tags": {
"language": "rus",
"handler_name": "DUB [iTunes]",
"vendor_id": "[0][0][0][0]"
}
},
{
"index": 2,
"codec_name": "bin_data",
"codec_long_name": "binary data",
"codec_type": "data",
"codec_tag_string": "text",
"codec_tag": "0x74786574",
"id": "0x3",
"r_frame_rate": "0/0",
"avg_frame_rate": "0/0",
"time_base": "1/1000",
"start_pts": 0,
"start_time": "0.000000",
"duration_ts": 369055,
"duration": "369.055000",
"bit_rate": "2",
"nb_frames": "3",
"extradata_size": 43,
"disposition": {
"default": 0,
"dub": 0,
"original": 0,
"comment": 0,
"lyrics": 0,
"karaoke": 0,
"forced": 0,
"hearing_impaired": 0,
"visual_impaired": 0,
"clean_effects": 0,
"attached_pic": 0,
"timed_thumbnails": 0,
"captions": 0,
"descriptions": 0,
"metadata": 0,
"dependent": 0,
"still_image": 0
},
"tags": {
"language": "eng",
"handler_name": "SubtitleHandler"
}
}
],
"format": {
"filename": "d:\\vids\\ero\\Язычницкий обряд оплодотворения девственницы Секс сцены из ф.mp4",
"nb_streams": 3,
"nb_programs": 0,
"format_name": "mov,mp4,m4a,3gp,3g2,mj2",
"format_long_name": "QuickTime / MOV",
"start_time": "0.000000",
"duration": "369.078000",
"size": "45621229",
"bit_rate": "988869",
"probe_score": 100,
"tags": {
"major_brand": "isom",
"minor_version": "512",
"compatible_brands": "isomiso2avc1mp41",
"title": "Midsommar.2019 #TorpedoUK#",
"encoder": "XviD4PSP 7.0.493 PRO",
"hd_video": "1",
"iTunMOVI": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n<plist version=\"1.0\">\n<dict>\n <key>encoding_info</key>\n <array>\n <dict>\n <key>name</key>\n <string><![CDATA[MP4 > MP4 YouTube 720p\nH264 > X264 CRF 21.0 HQ\nAAC-LC > AAC CBR 128kbps LC]]></string>\n </dict>\n </array>\n</dict>\n</plist>\n"
}
}
}

5
sea.session.sql
Unescape Escape View File

@ -1,2 +1,3 @@
select * from filemeta limit 10 select * from filemeta
order by id desc; order by id desc
limit 10;

115
test.jl
Unescape Escape View File

@ -1,4 +1,5 @@
# То же самое, но на julia # То же самое, но на julia
# Версия 1.0 (рабочая)
using Logging using Logging
using Printf using Printf
@ -36,7 +37,7 @@ end
function humanbitrate(s) function humanbitrate(s)
""" """
The function "humanbitrate" takes a string representing a bitrate in the format "a/b" and returns # The function "humanbitrate" takes a string representing a bitrate in the format "a/b" and returns
the bitrate as a float with one decimal place. the bitrate as a float with one decimal place.
:param s: The parameter "s" is a string that represents a fraction in the format "a/b", where "a" :param s: The parameter "s" is a string that represents a fraction in the format "a/b", where "a"
@ -79,6 +80,10 @@ function valid_file(filename)
return ext in valids return ext in valids
end end
function humandate(d)
return Dates.format(Dates.unix2datetime(d), "dd.mm.yyyy")
end
# file_info function returns information from file # file_info function returns information from file
function file_info(filename) function file_info(filename)
_, ext = splitext(filename) _, ext = splitext(filename)
@ -90,8 +95,8 @@ function file_info(filename)
"hsize" => humanbytes(info.size), "hsize" => humanbytes(info.size),
"mtime" => info.mtime, "mtime" => info.mtime,
"ctime" => info.ctime, "ctime" => info.ctime,
"hmtime" => Dates.format(Dates.unix2datetime(info.mtime), "dd.mm.yyyy"), "hmtime" => humandate(info.mtime),
"hctime" => Dates.format(Dates.unix2datetime(info.ctime), "dd.mm.yyyy"), "hctime" => humandate(info.ctime),
"ext" => ext, "ext" => ext,
"crc"=> Int(crc), "crc"=> Int(crc),
) )
@ -108,23 +113,39 @@ function get_video_stream(stream)
return -1 return -1
end end
# возвращает индекс аудио-потока, -1 при отсутствии
function get_audio_stream(stream)
for i in eachindex(stream)
if stream[i]["codec_type"] == "audio"
return i
end
end
return -1
end
# Потоки могут быть перепутаны, поэтму нужно сначала # Потоки могут быть перепутаны, поэтму нужно сначала
# определить индекс видео-потока # определить индекс видео-потока
function file_meta(filename) function file_meta(filename)
cmd = `ffprobe -v quiet -print_format json -show_format -show_streams "$filename"` cmd = `ffprobe -v quiet -print_format json -show_format -show_streams "$filename"`
out = read(cmd, String) data = read(cmd, String) |> JSON.parse
data = JSON.parse(out)
n = get_video_stream(data["streams"]) n = get_video_stream(data["streams"])
audio = get_audio_stream(data["streams"]) != -1
dur = parse(Float64, data["format"]["duration"]) dur = parse(Float64, data["format"]["duration"])
hdur = Dates.unix2datetime(dur) hdur = Dates.unix2datetime(dur)
hduration = Dates.format((hdur), "HH:MM:SS")
res = Dict( res = Dict(
"duration" => hdur, "duration" => hdur,
"hduration" => Dates.format((hdur), "HH:MM:SS"), "hduration" => hduration,
"width" => data["streams"][n]["width"], "width" => data["streams"][n]["width"],
"height" => data["streams"][n]["height"], "height" => data["streams"][n]["height"],
"bit_rate" => humanbitrate(data["streams"][n]["avg_frame_rate"]), "bit_rate" => humanbitrate(data["streams"][n]["avg_frame_rate"]),
"codec" => data["streams"][n]["codec_name"], "codec" => data["streams"][n]["codec_name"],
"audio" => audio,
"tags" => "",
"type" => hduration < "00:10:00" ? "clip" : "film",
"newname" => "",
"poster" => "",
# "aspect_ratio" => data["streams"][n]["display_aspect_ratio"], # "aspect_ratio" => data["streams"][n]["display_aspect_ratio"],
) )
return res return res
@ -139,32 +160,91 @@ function check_file(filename)
return result return result
end end
function main() function main(host_id::Integer)
dir = "d:\\torrent" dir = "d:\\vids\\ero\\2"
@info "Обработка каталога: $dir" @info "Обработка каталога: $dir"
files = readdir(dir, join=true) files = readdir(dir, join=true)
n = length(files) n = length(files)
println("всего: $n files") println("всего: $n files")
conn = DBInterface.connect(MySQL.Connection, "xigmanas", "itman", "X753951x", db="vid")
@time begin @time begin
@sync begin # @sync begin
for file in filter(valid_file, files) for file in filter(valid_file, files)
Threads.@spawn begin # Threads.@spawn begin
meta = check_file(file) meta = check_file(file)
println(meta) meta["host_id"] = host_id
# println(meta)
save_db(conn, meta)
# end
end end
# end
end end
println("обработано")
close(conn)
end end
function exist(conn, crc::Integer)
cmd = "select count(*) as cnt from filemeta where crc = $crc"
result = DBInterface.execute(conn, cmd)
data = first(result)
# println("data = $data")
return data.cnt > 0
end end
println("обработано") # сохраняет meta в БД
function save_db(conn, m)
filename = m["filename"]
if exist(conn, m["crc"])
@info "уже имеется: $filename"
else
println("сохраняю: $filename")
host_id = m["host_id"]
filename = replace(m["filename"], "\\" => "\\\\")
newname = m["newname"]
size = m["size"]
hsize = m["hsize"]
hduration = m["hduration"]
width = m["width"]
height = m["height"]
bitrate = m["bit_rate"]
ext = m["ext"]
crc = m["crc"]
hctime = m["hctime"]
hmtime = m["hmtime"]
poster = m["poster"]
audio = m["audio"]
codec = m["codec"]
tags = m["tags"]
type = m["type"]
fields = """
host_id, filename, newname, size,
hsize, duration, width,
height, bitrate, ext, crc,
ctime, mtime,
poster, audio, codec, tags,
type
"""
values = """
$host_id, '$filename', '$newname', $size,
'$hsize', '$hduration', $width,
$height, '$bitrate', '$ext', $crc,
'$hctime', '$hmtime',
'$poster', $audio, '$codec', '$tags',
'$type'
"""
cmd = "insert into filemeta ($fields) values ($values)"
println("file: ", filename)
result = DBInterface.execute(conn, cmd)
# println("result: ", result)
end
end end
function openDB() function openDB()
conn = DBInterface.connect(MySQL.Connection, "xigmanas", "itman", "X753951x", db="sea") conn = DBInterface.connect(MySQL.Connection, "xigmanas", "itman", "X753951x", db="sea")
# conn.db = "sea"
# println("DB: ", conn)
limit = 2 limit = 2
ext = ".webp" ext = ".webp"
cmd = "select ext, newurl from filemeta where ext = '$ext' limit $limit" cmd = "select ext, newurl from filemeta where ext = '$ext' limit $limit"
@ -175,7 +255,10 @@ function openDB()
end end
end end
# -------- основная программа -------------- #=
-------- основная программа --------------
=#
# parallel_execution() # parallel_execution()
# openDB() # openDB()
main() main(1)

Write Preview
Loading…
Cancel
Save