deviantart/types/
scraped_webpage_info.rs

1use super::Deviation;
2use super::Media;
3use once_cell::sync::Lazy;
4use regex::Regex;
5use std::collections::HashMap;
6use url::Url;
7
8/// An error that may occur while parsing a [`ScrapedWebPageInfo`] from a html string.
9#[derive(Debug, thiserror::Error)]
10pub enum FromHtmlStrError {
11    /// Missing the InitialState variable
12    #[error("missing initial state")]
13    MissingInitialState,
14
15    /// Failed to parse some state
16    #[error(transparent)]
17    InvalidJson(#[from] serde_json::Error),
18}
19
20/// Info scraped from a deviation url
21#[derive(Debug, serde::Deserialize)]
22pub struct ScrapedWebPageInfo {
23    /// Page config like csrf tokens
24    #[serde(rename = "@@config")]
25    pub config: Config,
26
27    /// Deviations extended deviations maybe?
28    #[serde(rename = "@@entities")]
29    pub entities: Option<Entities>,
30
31    /// ?
32    #[serde(rename = "@@DUPERBROWSE")]
33    pub duper_browse: Option<DuperBrowse>,
34
35    /// Info about the current session
36    #[serde(rename = "@@publicSession")]
37    pub public_session: PublicSession,
38
39    /// Streams
40    #[serde(rename = "@@streams")]
41    pub streams: Option<Streams>,
42
43    /// Needed for login.
44    ///
45    /// Note that this is a different csrf token from the config struct.
46    #[serde(rename = "csrfToken")]
47    pub csrf_token: Option<Box<str>>,
48
49    #[serde(rename = "gallectionSection")]
50    pub gallection_section: Option<GallectionSection>,
51
52    /// Needed for login.
53    #[serde(rename = "luToken")]
54    pub lu_token: Option<Box<str>>,
55
56    /// Needed for login.
57    #[serde(rename = "luToken2")]
58    pub lu_token2: Option<Box<str>>,
59
60    /// Unknown data
61    #[serde(flatten)]
62    pub unknown: HashMap<String, serde_json::Value>,
63}
64
65impl ScrapedWebPageInfo {
66    /// Parse this from a html string
67    pub fn from_html_str(input: &str) -> Result<Self, FromHtmlStrError> {
68        static REGEX: Lazy<Regex> = Lazy::new(|| {
69            Regex::new(r#"window\.__INITIAL_STATE__ = JSON\.parse\("(.*)"\);"#).unwrap()
70        });
71
72        let capture = REGEX
73            .captures(input)
74            .and_then(|captures| captures.get(1))
75            .ok_or(FromHtmlStrError::MissingInitialState)?;
76        // TODO: Escape properly
77        let capture = capture
78            .as_str()
79            .replace("\\\"", "\"")
80            .replace("\\'", "'")
81            .replace("\\\\", "\\");
82
83        // std::fs::write("out.json", capture.as_str()).unwrap();
84
85        Ok(serde_json::from_str(&capture)?)
86    }
87
88    /// Returns `true` if logged in
89    pub fn is_logged_in(&self) -> bool {
90        self.public_session.is_logged_in
91    }
92
93    /// Get the current deviation's id
94    pub fn get_current_deviation_id(&self) -> Option<&serde_json::Value> {
95        Some(
96            &self
97                .duper_browse
98                .as_ref()?
99                .root_stream
100                .as_ref()?
101                .current_open_item,
102        )
103    }
104
105    /// Get the [`Deviation`] for this page.
106    pub fn get_current_deviation(&self) -> Option<&Deviation> {
107        let id = self.get_current_deviation_id()?;
108        let id = match id {
109            serde_json::Value::Number(n) => n.as_u64()?,
110            serde_json::Value::String(s) => s.parse().ok()?,
111            _ => return None,
112        };
113        self.get_deviation_by_id(id)
114    }
115
116    /// Get the [`DeviationExtended`] for this page.
117    pub fn get_current_deviation_extended(&self) -> Option<&DeviationExtended> {
118        let id = self.get_current_deviation_id()?;
119        let mut key_buffer = itoa::Buffer::new();
120        let key = match id {
121            serde_json::Value::Number(n) => {
122                let n = n.as_u64()?;
123                key_buffer.format(n)
124            }
125            serde_json::Value::String(s) => s,
126            _ => return None,
127        };
128        self.entities
129            .as_ref()?
130            .deviation_extended
131            .as_ref()?
132            .get(key)
133    }
134
135    /// Get a deviation by id, if it exists
136    pub fn get_deviation_by_id(&self, id: u64) -> Option<&Deviation> {
137        let mut key_buffer = itoa::Buffer::new();
138        self.entities.as_ref()?.deviation.get(key_buffer.format(id))
139    }
140
141    /// Take a deviation by id, if it exists
142    pub fn take_deviation_by_id(&mut self, id: u64) -> Option<Deviation> {
143        let mut key_buffer = itoa::Buffer::new();
144        self.entities
145            .as_mut()?
146            .deviation
147            .remove(key_buffer.format(id))
148    }
149
150    /// Get the current folder id, if in a gallery.
151    pub fn get_current_folder_id(&self) -> Option<i64> {
152        Some(self.gallection_section.as_ref()?.selected_folder_id)
153    }
154
155    /// Get a stream for folder post ids, by folder id.
156    ///
157    /// This will return the deviation ids for the current folder.
158    pub fn get_folder_deviations_stream(&self, folder_id: i64) -> Option<&WithOffsetStream> {
159        let key = format!("folder-deviations-gallery-{folder_id}");
160
161        self.streams
162            .as_ref()?
163            .streams
164            .get(&key)?
165            .as_with_offset_stream()
166    }
167
168    /// Get a gallery folder entity by id
169    pub fn get_gallery_folder_entity(&self, folder_id: i64) -> Option<&GalleryFolder> {
170        self.entities
171            .as_ref()?
172            .gallery_folder
173            .as_ref()?
174            .get(itoa::Buffer::new().format(folder_id))
175    }
176
177    /// Get a user entity by id
178    pub fn get_user_entity(&self, user_id: u64) -> Option<&User> {
179        self.entities
180            .as_ref()?
181            .user
182            .as_ref()?
183            .get(itoa::Buffer::new().format(user_id))
184    }
185}
186
187/// ?
188#[derive(Debug, serde::Deserialize)]
189pub struct Config {
190    /// The page's csrf token
191    #[serde(rename = "csrfToken")]
192    pub csrf_token: String,
193
194    /// Unknown data
195    #[serde(flatten)]
196    pub unknown: HashMap<String, serde_json::Value>,
197}
198
199/// Page entities, like deviations, folders, and users.
200#[derive(Debug, serde::Deserialize)]
201pub struct Entities {
202    /// Deviations
203    pub deviation: HashMap<String, Deviation>,
204
205    /// Extended Deviation Info
206    #[serde(rename = "deviationExtended")]
207    pub deviation_extended: Option<HashMap<String, DeviationExtended>>,
208
209    /// Gallery folders
210    #[serde(rename = "galleryFolder")]
211    pub gallery_folder: Option<HashMap<String, GalleryFolder>>,
212
213    /// Users
214    pub user: Option<HashMap<String, User>>,
215
216    /// Unknown data
217    #[serde(flatten)]
218    pub unknown: HashMap<String, serde_json::Value>,
219}
220
221/// Extended Info about a deviation
222#[derive(Debug, serde::Deserialize)]
223pub struct DeviationExtended {
224    /// Download info
225    pub download: Option<Download>,
226
227    /// HTML description
228    pub description: Option<String>,
229
230    /// Other media for this deviation
231    #[serde(rename = "additionalMedia")]
232    pub additional_media: Option<Vec<AdditionalMedia>>,
233
234    /// The id of the deviation this belongs to.
235    #[serde(rename = "parentDeviationEntityId")]
236    pub parent_deviation_entity_id: u64,
237
238    /// Unknown data
239    #[serde(flatten)]
240    pub unknown: HashMap<String, serde_json::Value>,
241}
242
243impl DeviationExtended {
244    /// Check if additional media is downloadable.
245    ///
246    /// Older additionalMedia deviation images can be downloaded.
247    /// Newer ones cannot.
248    ///
249    /// See: https://github.com/mikf/gallery-dl/issues/6653#issuecomment-2816585744
250    pub fn can_download_additional_media(&self) -> bool {
251        // Return false if there is no additional media.
252        let additional_media = match self.additional_media.as_ref() {
253            Some(additional_media) => additional_media,
254            None => return false,
255        };
256
257        // Old deviations can always be downloaded.
258        // ID determined experimentally.
259        if self.parent_deviation_entity_id < 1184619292 {
260            return true;
261        }
262
263        // The second token is for downloads.
264        // If present, we can download.
265        additional_media
266            .iter()
267            .all(|entry| entry.media.token.len() > 1)
268    }
269}
270
271/// A gallery folder
272#[derive(Debug, serde::Deserialize)]
273pub struct GalleryFolder {
274    /// The folder id.
275    ///
276    /// For some reason, this can be -1 sometimes.
277    #[serde(rename = "folderId")]
278    pub folder_id: i64,
279
280    /// The name of the folder
281    pub name: String,
282
283    /// The user id of the owner of the folder
284    pub owner: u64,
285
286    /// Unknown data
287    #[serde(flatten)]
288    pub unknown: HashMap<String, serde_json::Value>,
289}
290
291/// A user
292#[derive(Debug, serde::Deserialize)]
293pub struct User {
294    /// The user id
295    #[serde(rename = "userId")]
296    pub user_id: u64,
297
298    /// The user name
299    pub username: String,
300
301    /// Unknown data
302    #[serde(flatten)]
303    pub unknown: HashMap<String, serde_json::Value>,
304}
305
306#[derive(Debug, serde::Deserialize)]
307pub struct Download {
308    /// The file size
309    pub filesize: u64,
310
311    /// The image height
312    pub height: u32,
313
314    /// The image width
315    pub width: u32,
316
317    /// ?
318    #[serde(rename = "type")]
319    pub kind: String,
320
321    /// The url
322    pub url: Url,
323
324    /// Unknown data
325    #[serde(flatten)]
326    pub unknown: HashMap<String, serde_json::Value>,
327}
328
329#[derive(Debug, serde::Deserialize)]
330pub struct AdditionalMedia {
331    /// Media info
332    pub media: Media,
333
334    /// Unknown data
335    #[serde(flatten)]
336    pub unknown: HashMap<String, serde_json::Value>,
337}
338
339/// ?
340#[derive(Debug, serde::Deserialize)]
341pub struct DuperBrowse {
342    /// ?
343    #[serde(rename = "rootStream")]
344    pub root_stream: Option<RootStream>,
345
346    /// Unknown data
347    #[serde(flatten)]
348    pub unknown: HashMap<String, serde_json::Value>,
349}
350
351/// ?
352#[derive(Debug, serde::Deserialize)]
353pub struct RootStream {
354    /// The id of the current deviation. This is either a number or string.
355    #[serde(rename = "currentOpenItem")]
356    pub current_open_item: serde_json::Value,
357
358    /// Unknown data
359    #[serde(flatten)]
360    pub unknown: HashMap<String, serde_json::Value>,
361}
362
363/// ?
364#[derive(Debug, serde::Deserialize)]
365pub struct PublicSession {
366    /// Whether the user is logged in
367    #[serde(rename = "isLoggedIn")]
368    pub is_logged_in: bool,
369
370    /// Unknown data
371    #[serde(flatten)]
372    pub unknown: HashMap<String, serde_json::Value>,
373}
374
375/// The streams field
376#[derive(Debug, serde::Deserialize)]
377pub struct Streams {
378    /// Search results appear here
379    #[serde(rename = "@@BROWSE_PAGE_STREAM")]
380    pub browse_page_stream: Option<BrowsePageStream>,
381
382    /// Extra data.
383    ///
384    /// This can include data whos purpose is known, like entries in a folder.
385    #[serde(flatten)]
386    pub streams: HashMap<String, Stream>,
387}
388
389/// ?
390#[derive(Debug, serde::Deserialize)]
391#[serde(tag = "streamType")]
392pub enum Stream {
393    #[serde(rename = "WITH_OFFSET")]
394    WithOffset(WithOffsetStream),
395
396    #[serde(untagged)]
397    Unknown(serde_json::Value),
398}
399
400impl Stream {
401    /// Get this as a WithOffset stream.
402    pub fn as_with_offset_stream(&self) -> Option<&WithOffsetStream> {
403        match self {
404            Self::WithOffset(stream) => Some(stream),
405            _ => None,
406        }
407    }
408}
409
410/// ?
411#[derive(Debug, serde::Deserialize)]
412pub struct WithOffsetStream {
413    /// Items in the stream?
414    pub items: Vec<u64>,
415
416    /// The # of items per fetch?
417    #[serde(rename = "itemsPerFetch")]
418    pub items_per_fetch: u32,
419
420    /// Has more entries?
421    #[serde(rename = "hasMore")]
422    pub has_more: bool,
423
424    /// ?
425    #[serde(rename = "hasLess")]
426    pub has_less: bool,
427
428    /// Unknown data
429    #[serde(flatten)]
430    pub unknown: HashMap<String, serde_json::Value>,
431}
432
433/// Search results appear here
434#[derive(Debug, serde::Deserialize)]
435pub struct BrowsePageStream {
436    /// The cursor
437    pub cursor: String,
438
439    /// Whether this has less?
440    #[serde(rename = "hasLess")]
441    pub has_less: bool,
442
443    /// Whether this has more?
444    #[serde(rename = "hasMore")]
445    pub has_more: bool,
446
447    /// Deviation ids?
448    ///
449    /// Usually, these are integers representing deviation ids.
450    /// In some cases, these are strings of the format "xx-nnnnn",
451    /// where the "xx" part is unknown and the "nnnnn" part is a deviation id.
452    pub items: Vec<serde_json::Value>,
453
454    /// The # of items per page
455    #[serde(rename = "itemsPerFetch")]
456    pub items_per_fetch: u64,
457
458    /// Stream Params
459    #[serde(rename = "streamParams")]
460    pub stream_params: StreamParams,
461
462    /// The stream type
463    #[serde(rename = "streamType")]
464    pub stream_type: String,
465
466    /// The stream id
467    #[serde(rename = "streamId")]
468    pub stream_id: String,
469
470    /// ?
471    #[serde(rename = "fetchNextCallback")]
472    pub fetch_next_callback: String,
473
474    /// Unknown data
475    #[serde(flatten)]
476    pub unknown: HashMap<String, serde_json::Value>,
477}
478
479/// Stream params
480#[derive(Debug, serde::Deserialize)]
481pub struct StreamParams {
482    /// Request params
483    ///
484    /// This can be at least either a string or a bool.
485    /// "include_hreflangs": true
486    #[serde(rename = "requestParams")]
487    pub request_params: HashMap<String, serde_json::Value>,
488
489    /// ?
490    #[serde(rename = "itemType")]
491    pub item_type: String,
492
493    /// ?
494    #[serde(rename = "requestEndpoint")]
495    pub request_endpoint: String,
496
497    /// Unknown data
498    #[serde(flatten)]
499    pub unknown: HashMap<String, serde_json::Value>,
500}
501
502/// Gallery selection info
503#[derive(Debug, serde::Deserialize)]
504pub struct GallectionSection {
505    /// The id of the selected folder
506    #[serde(rename = "selectedFolderId")]
507    pub selected_folder_id: i64,
508
509    /// Unknown data
510    #[serde(flatten)]
511    pub unknown: HashMap<String, serde_json::Value>,
512}
513
514#[cfg(test)]
515mod test {
516    use super::*;
517
518    const SCRAPED_WEBPAGE: &str = include_str!("../../test_data/scraped_webpage.json");
519    const LOGIN_WEBPAGE: &str = include_str!("../../test_data/login_webpage.json");
520
521    #[test]
522    fn parse_scraped_webpage() {
523        let scraped_webpage_info: ScrapedWebPageInfo =
524            serde_json::from_str(SCRAPED_WEBPAGE).expect("failed to parse scraped webpage info");
525        assert_eq!(
526            scraped_webpage_info
527                .get_current_deviation_id()
528                .expect("missing current deviation id"),
529            119577071
530        );
531        // dbg!(scraped_deviation_info.entities.deviation);
532    }
533
534    #[test]
535    fn parse_login_webpage() {
536        let _scraped_webpage_info: ScrapedWebPageInfo =
537            serde_json::from_str(LOGIN_WEBPAGE).expect("failed to parse scraped webpage info");
538    }
539}
deviantart/types/scraped_webpage_info.rs

deviantart/types/
scraped_webpage_info.rs
