deviantart/
client.rs

1use crate::Deviation;
2use crate::Error;
3use crate::ListFolderContentsResponse;
4use crate::OEmbed;
5use crate::ScrapedWebPageInfo;
6use crate::WrapBoxError;
7use reqwest::header::HeaderMap;
8use reqwest::header::HeaderValue;
9use reqwest_cookie_store::CookieStoreMutex;
10use std::sync::Arc;
11use url::Url;
12
13const USER_AGENT_STR: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
14static ACCEPT_LANGUAGE_VALUE: HeaderValue = HeaderValue::from_static("en,en-US;q=0,5");
15static ACCEPT_VALUE: HeaderValue = HeaderValue::from_static("*/*");
16static REFERER_VALUE: HeaderValue = HeaderValue::from_static(HOME_URL);
17
18const HOME_URL: &str = "https://www.deviantart.com/";
19const LOGIN_URL: &str = "https://www.deviantart.com/users/login";
20
21/// A DeviantArt Client
22#[derive(Debug, Clone)]
23pub struct Client {
24    /// The inner http client.
25    ///
26    /// You probably shouldn't touch this.
27    pub client: reqwest::Client,
28
29    /// The cookie store.
30    pub cookie_store: Arc<CookieStoreMutex>,
31}
32
33impl Client {
34    /// Make a new [`Client`].
35    pub fn new() -> Self {
36        Self::new_with_user_agent(USER_AGENT_STR)
37    }
38
39    /// Make a new [`Client`] with the given user agent.
40    pub fn new_with_user_agent(user_agent: &str) -> Self {
41        let mut default_headers = HeaderMap::new();
42        default_headers.insert(
43            reqwest::header::ACCEPT_LANGUAGE,
44            ACCEPT_LANGUAGE_VALUE.clone(),
45        );
46        default_headers.insert(reqwest::header::ACCEPT, ACCEPT_VALUE.clone());
47        default_headers.insert(reqwest::header::REFERER, REFERER_VALUE.clone());
48
49        let cookie_store = Arc::new(CookieStoreMutex::new(Default::default()));
50        let client = reqwest::Client::builder()
51            .cookie_provider(cookie_store.clone())
52            .user_agent(user_agent)
53            .default_headers(default_headers)
54            .build()
55            .expect("failed to build deviantart client");
56
57        Client {
58            client,
59            cookie_store,
60        }
61    }
62
63    /// Load the cookie store from a json reader.
64    pub async fn load_json_cookies<R>(&self, reader: R) -> Result<(), Error>
65    where
66        R: std::io::BufRead + Send + 'static,
67    {
68        let cookie_store = self.cookie_store.clone();
69        tokio::task::spawn_blocking(move || {
70            let new_cookie_store = cookie_store::serde::json::load(reader)
71                .map_err(|e| Error::CookieStore(WrapBoxError(e)))?;
72            let mut cookie_store = cookie_store.lock().expect("cookie store is poisoned");
73            *cookie_store = new_cookie_store;
74            Ok(())
75        })
76        .await?
77    }
78
79    /// Save the cookie store from a json writer.
80    pub async fn save_json_cookies<W>(&self, mut writer: W) -> Result<W, Error>
81    where
82        W: std::io::Write + Send + 'static,
83    {
84        let cookie_store = self.cookie_store.clone();
85        tokio::task::spawn_blocking(move || {
86            let cookie_store = cookie_store.lock().expect("cookie store is poisoned");
87            cookie_store::serde::json::save(&cookie_store, &mut writer)
88                .map_err(|e| Error::CookieStore(WrapBoxError(e)))?;
89            Ok(writer)
90        })
91        .await?
92    }
93
94    /// Scrape a webpage for info.
95    pub async fn scrape_webpage(&self, url: &str) -> Result<ScrapedWebPageInfo, Error> {
96        let text = self
97            .client
98            .get(url)
99            .send()
100            .await?
101            .error_for_status()?
102            .text()
103            .await?;
104
105        let scraped_webpage =
106            tokio::task::spawn_blocking(move || ScrapedWebPageInfo::from_html_str(&text)).await??;
107
108        Ok(scraped_webpage)
109    }
110
111    /// Login to get access to more results from apis.
112    ///
113    /// This will also clean the cookie jar.
114    pub async fn login(&self, username: &str, password: &str) -> Result<(), Error> {
115        // Clean the jar of expired cookies
116        {
117            let mut cookie_store = self.cookie_store.lock().expect("cookie store is poisoned");
118
119            // We need to allocate here as the cookie_store iter cannot be alive when we try to remove items from the cookie store.
120            let to_remove: Vec<_> = cookie_store
121                .iter_any()
122                .filter(|cookie| cookie.is_expired())
123                .map(|cookie| {
124                    let domain = cookie.domain().unwrap_or("");
125                    let name = cookie.name();
126                    let path = cookie.path().unwrap_or("");
127
128                    (domain.to_string(), name.to_string(), path.to_string())
129                })
130                .collect();
131
132            for (domain, name, path) in to_remove {
133                cookie_store.remove(&domain, &name, &path);
134            }
135        }
136
137        // Initial req to login page.
138        let login_page = self.scrape_webpage(LOGIN_URL).await?;
139        let login_page_csrf_token = login_page
140            .csrf_token
141            .as_deref()
142            .ok_or(Error::MissingField { name: "csrfToken" })?;
143        let login_page_lu_token = login_page
144            .lu_token
145            .as_deref()
146            .ok_or(Error::MissingField { name: "luToken" })?;
147
148        // Get the password input page.
149        // The username and password inputs are on different pages.
150        let password_page_text = self
151            .client
152            .post("https://www.deviantart.com/_sisu/do/step2")
153            .form(&[
154                ("referer", LOGIN_URL),
155                ("referer_type", ""),
156                ("csrf_token", login_page_csrf_token),
157                ("challenge", "0"),
158                ("lu_token", login_page_lu_token),
159                ("username", username),
160                ("remember", "on"),
161            ])
162            .send()
163            .await?
164            .error_for_status()?
165            .text()
166            .await?;
167        let password_page = tokio::task::spawn_blocking(move || {
168            ScrapedWebPageInfo::from_html_str(&password_page_text)
169        })
170        .await??;
171        let password_page_csrf_token = password_page
172            .csrf_token
173            .as_deref()
174            .ok_or(Error::MissingField { name: "csrfToken" })?;
175        let password_page_lu_token = password_page
176            .lu_token
177            .as_deref()
178            .ok_or(Error::MissingField { name: "luToken" })?;
179        let password_page_lu_token2 = password_page
180            .lu_token2
181            .as_deref()
182            .ok_or(Error::MissingField { name: "luToken2" })?;
183
184        // Submit password
185        let signin_url = "https://www.deviantart.com/_sisu/do/signin";
186        let response = self
187            .client
188            .post(signin_url)
189            .form(&[
190                ("referer", signin_url),
191                ("referer_type", ""),
192                ("csrf_token", password_page_csrf_token),
193                ("challenge", "0"),
194                ("lu_token", password_page_lu_token),
195                ("lu_token2", password_page_lu_token2),
196                ("username", ""),
197                ("password", password),
198                ("remember", "on"),
199            ])
200            .send()
201            .await?
202            .error_for_status()?;
203
204        let text = response.text().await?;
205        let scraped_webpage =
206            tokio::task::spawn_blocking(move || ScrapedWebPageInfo::from_html_str(&text)).await??;
207        if !scraped_webpage.is_logged_in() {
208            return Err(Error::SignInFailed);
209        }
210
211        Ok(())
212    }
213
214    /// Run a GET request on the home page and check if the user is logged in
215    pub async fn is_logged_in_online(&self) -> Result<bool, Error> {
216        Ok(self.scrape_webpage(HOME_URL).await?.is_logged_in())
217    }
218
219    /// OEmbed API
220    pub async fn get_oembed(&self, url: &str) -> Result<OEmbed, Error> {
221        let url = Url::parse_with_params("https://backend.deviantart.com/oembed", &[("url", url)])?;
222        let res = self
223            .client
224            .get(url.as_str())
225            .send()
226            .await?
227            .error_for_status()?
228            .json()
229            .await?;
230        Ok(res)
231    }
232
233    /// Run a search using the low level api
234    pub async fn search_raw(
235        &self,
236        query: &str,
237        cursor: Option<&str>,
238    ) -> Result<ScrapedWebPageInfo, Error> {
239        let mut url = Url::parse_with_params("https://www.deviantart.com/search", &[("q", query)])?;
240        {
241            let mut query_pairs = url.query_pairs_mut();
242            if let Some(cursor) = cursor {
243                query_pairs.append_pair("cursor", cursor);
244            }
245        }
246
247        self.scrape_webpage(url.as_str()).await
248    }
249
250    /// Run a search
251    pub fn search(&self, query: &str, cursor: Option<&str>) -> SearchCursor {
252        SearchCursor::new(self.clone(), query, cursor)
253    }
254
255    /// List gallery contents.
256    ///
257    /// A folder_id of -1 means the All folder.
258    pub async fn list_folder_contents(
259        &self,
260        username: &str,
261        folder_id: i64,
262        offset: u64,
263        csrf_token: &str,
264    ) -> Result<ListFolderContentsResponse, Error> {
265        let mut url = Url::parse("https://www.deviantart.com/_puppy/dashared/gallection/contents")?;
266        {
267            let mut query_pairs = url.query_pairs_mut();
268
269            query_pairs.append_pair("username", username);
270            query_pairs.append_pair("type", "gallery");
271            query_pairs.append_pair("order", "personalized");
272            query_pairs.append_pair("offset", itoa::Buffer::new().format(offset));
273            query_pairs.append_pair("limit", "24");
274            if folder_id == -1 {
275                query_pairs.append_pair("all_folder", "true");
276            } else {
277                query_pairs.append_pair("folderid", itoa::Buffer::new().format(folder_id));
278            }
279            query_pairs.append_pair("csrf_token", csrf_token);
280        }
281
282        Ok(self
283            .client
284            .get(url)
285            .send()
286            .await?
287            .error_for_status()?
288            .json()
289            .await?)
290    }
291}
292
293impl Default for Client {
294    fn default() -> Self {
295        Self::new()
296    }
297}
298
299#[derive(Debug)]
300pub struct SearchCursor {
301    /// The client
302    client: Client,
303
304    /// The current page
305    page: Option<ScrapedWebPageInfo>,
306
307    /// the query
308    query: String,
309    /// the cursor
310    cursor: Option<String>,
311}
312
313impl SearchCursor {
314    /// Make a new Search Cursor
315    pub fn new(client: Client, query: &str, cursor: Option<&str>) -> Self {
316        Self {
317            client,
318
319            page: None,
320
321            query: query.into(),
322            cursor: cursor.map(|c| c.into()),
323        }
324    }
325
326    /// Get the current page of deviations
327    pub fn current_deviations(&self) -> Option<Result<Vec<&Deviation>, Error>> {
328        let page = self.page.as_ref()?;
329
330        let browse_page_stream = page
331            .streams
332            .as_ref()
333            .unwrap()
334            .browse_page_stream
335            .as_ref()
336            .unwrap();
337
338        Some(
339            browse_page_stream
340                .items
341                .iter()
342                .filter_map(|id| {
343                    // TODO: Investigate string format more.
344                    id.as_u64()
345                })
346                .map(|id| {
347                    page.get_deviation_by_id(id)
348                        .ok_or(Error::MissingDeviation(id))
349                })
350                .collect(),
351        )
352    }
353
354    /// Take the current page of deviations
355    pub fn take_current_deviations(&mut self) -> Option<Result<Vec<Deviation>, Error>> {
356        let mut page = self.page.take()?;
357
358        let browse_page_stream = page
359            .streams
360            .as_mut()
361            .unwrap()
362            .browse_page_stream
363            .as_mut()
364            .unwrap();
365
366        let items = std::mem::take(&mut browse_page_stream.items);
367        Some(
368            items
369                .iter()
370                .filter_map(|id| {
371                    // TODO: Investigate string format more.
372                    id.as_u64()
373                })
374                .map(|id| {
375                    page.take_deviation_by_id(id)
376                        .ok_or(Error::MissingDeviation(id))
377                })
378                .collect(),
379        )
380    }
381
382    /// Get the next page, updating the internal cursor.
383    pub async fn next_page(&mut self) -> Result<(), Error> {
384        let page = self
385            .client
386            .search_raw(&self.query, self.cursor.as_deref())
387            .await?;
388        // Validate before storing
389        match page
390            .streams
391            .as_ref()
392            .ok_or(Error::MissingStreams)?
393            .browse_page_stream
394            .as_ref()
395        {
396            Some(browse_page_stream) => {
397                self.cursor = Some(browse_page_stream.cursor.clone());
398            }
399            None => {
400                return Err(Error::MissingBrowsePageStream);
401            }
402        }
403        self.page = Some(page);
404
405        Ok(())
406    }
407}
408
409#[cfg(test)]
410mod test {
411    use super::*;
412
413    /// The default test config path
414    ///
415    /// Update this if this crate is moved to a different directory relative to the workspace Cargo.toml.
416    const DEFAULT_CONFIG_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../", "config.json");
417
418    #[derive(serde::Deserialize)]
419    struct Config {
420        username: String,
421        password: String,
422    }
423
424    impl Config {
425        fn from_path(path: &str) -> Option<Config> {
426            let file = match std::fs::read(path) {
427                Ok(file) => file,
428                Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
429                    return None;
430                }
431                Err(error) => panic!("failed to read file: {error}"),
432            };
433            let config = serde_json::from_reader(file.as_slice()).expect("failed to parse config");
434            Some(config)
435        }
436
437        fn from_env() -> Option<Self> {
438            let username = std::env::var_os("DEVIANTART_RS_USERNAME")?
439                .into_string()
440                .expect("invalid `DEVIANTART_RS_USERNAME`");
441            let password = std::env::var_os("DEVIANTART_RS_PASSWORD")?
442                .into_string()
443                .expect("invalid `DEVIANTART_RS_PASSWORD`");
444
445            Some(Self { username, password })
446        }
447
448        fn from_any(path: &str) -> Self {
449            Self::from_env()
450                .or_else(|| Self::from_path(path))
451                .expect("failed to load config from env or path")
452        }
453    }
454
455    #[tokio::test]
456    #[ignore]
457    async fn scrape_deviation() {
458        let client = Client::new();
459        let _scraped_webpage = client
460            .scrape_webpage("https://www.deviantart.com/zilla774/art/chaos-gerbil-RAWR-119577071")
461            .await
462            .expect("failed to scrape webpage");
463    }
464
465    #[tokio::test]
466    #[ignore]
467    async fn login_works() {
468        let config: Config = Config::from_any(DEFAULT_CONFIG_PATH);
469
470        let client = Client::new();
471        client
472            .login(&config.username, &config.password)
473            .await
474            .expect("failed to login");
475        let is_online = client
476            .is_logged_in_online()
477            .await
478            .expect("failed to check if online");
479        assert!(is_online);
480    }
481
482    #[tokio::test]
483    #[ignore]
484    async fn scrape_webpage_literature() {
485        let url =
486            "https://www.deviantart.com/tohokari-steel/art/A-Fictorian-Tale-Chapter-11-879180914";
487
488        let client = Client::new();
489        let scraped_webpage = client
490            .scrape_webpage(url)
491            .await
492            .expect("failed to scrape webpage");
493        let current_deviation = scraped_webpage
494            .get_current_deviation()
495            .expect("missing current deviation");
496        let text_content = current_deviation
497            .text_content
498            .as_ref()
499            .expect("missing text content");
500        let _markup = text_content
501            .html
502            .get_markup()
503            .expect("missing markup")
504            .expect("failed to parse markup");
505        // dbg!(&markup);
506    }
507
508    #[tokio::test]
509    #[ignore]
510    async fn scrape_webpage_gallery_folder() {
511        let url = "https://www.deviantart.com/tohokari-steel/gallery/91687487/prince-of-heart";
512
513        let client = Client::new();
514        let scraped_webpage = client
515            .scrape_webpage(url)
516            .await
517            .expect("failed to scrape webpage");
518        let folder_id = scraped_webpage
519            .get_current_folder_id()
520            .expect("missing folder id");
521        assert!(folder_id == 91687487, "{folder_id} != 91687487");
522
523        let stream = scraped_webpage
524            .get_folder_deviations_stream(folder_id)
525            .expect("missing stream");
526        assert!(stream.has_more);
527
528        let gallery_folder = scraped_webpage
529            .get_gallery_folder_entity(folder_id)
530            .expect("missing gallery folder entity");
531        dbg!(gallery_folder);
532
533        let user = scraped_webpage
534            .get_user_entity(gallery_folder.owner)
535            .expect("failed to get user");
536
537        let response = client
538            .list_folder_contents(
539                &user.username,
540                folder_id,
541                0,
542                &scraped_webpage.config.csrf_token,
543            )
544            .await
545            .expect("failed to list folder contents");
546        dbg!(response);
547    }
548
549    #[tokio::test]
550    #[ignore]
551    async fn scrape_webpage_gallery_all() {
552        let url = "https://www.deviantart.com/tohokari-steel/gallery";
553
554        let client = Client::new();
555        let scraped_webpage = client
556            .scrape_webpage(url)
557            .await
558            .expect("failed to scrape webpage");
559
560        let folder_id = scraped_webpage
561            .get_current_folder_id()
562            .expect("missing folder id");
563        assert!(folder_id == -1, "{folder_id} != -1");
564
565        let stream = scraped_webpage
566            .get_folder_deviations_stream(folder_id)
567            .expect("missing stream");
568        assert!(stream.has_more);
569
570        let gallery_folder = scraped_webpage
571            .get_gallery_folder_entity(folder_id)
572            .expect("missing gallery folder entity");
573        dbg!(gallery_folder);
574
575        let user = scraped_webpage
576            .get_user_entity(gallery_folder.owner)
577            .expect("failed to get user");
578
579        let response = client
580            .list_folder_contents(
581                &user.username,
582                folder_id,
583                0,
584                &scraped_webpage.config.csrf_token,
585            )
586            .await
587            .expect("failed to list folder contents");
588        dbg!(response);
589    }
590
591    // This is broken of CI.
592    // DeviantArt has probably blacklist GH's ips.
593    #[tokio::test]
594    #[ignore]
595    async fn oembed_works() {
596        let url =
597            "https://www.deviantart.com/tohokari-steel/art/A-Fictorian-Tale-Chapter-11-879180914";
598
599        let client = Client::new();
600        let oembed = client.get_oembed(url).await.expect("failed to get oembed");
601        assert!(oembed.title == "A Fictorian Tale Chapter 11");
602    }
603
604    #[tokio::test]
605    #[ignore]
606    async fn scrape_stash_info_works() {
607        let client = Client::new();
608        let url = "https://sta.sh/02bhirtp3iwq";
609        let stash = client
610            .scrape_webpage(url)
611            .await
612            .expect("failed to scrape stash");
613        let current_deviation_id = stash
614            .get_current_deviation_id()
615            .expect("missing current deviation id");
616        assert!(current_deviation_id.as_u64() == Some(590293385));
617    }
618
619    #[tokio::test]
620    #[ignore]
621    async fn it_works() {
622        let client = Client::new();
623        let mut search_cursor = client.search("sun", None);
624        search_cursor
625            .next_page()
626            .await
627            .expect("failed to get next page");
628        let results = search_cursor
629            .current_deviations()
630            .expect("missing page")
631            .expect("failed to look up deviations");
632        let first = &results.first().expect("no results");
633
634        let url = first
635            .get_download_url()
636            .or_else(|| first.get_fullview_url())
637            .expect("failed to find download url");
638        let bytes = client
639            .client
640            .get(url)
641            .send()
642            .await
643            .expect("failed to send")
644            .error_for_status()
645            .expect("bad status")
646            .bytes()
647            .await
648            .expect("failed to buffer bytes");
649
650        std::fs::write("test.jpg", &bytes).expect("failed to write to file");
651        search_cursor
652            .next_page()
653            .await
654            .expect("failed to get next page");
655        let _results = search_cursor
656            .current_deviations()
657            .expect("missing page")
658            .expect("failed to look up deviations");
659    }
660}
deviantart/client.rs

deviantart/
client.rs
