[Python] 네이버 스토어 찜 크롤링 with requests

데이터/크롤링

[Python] 네이버 스토어 찜 크롤링 with requests _2탄

Jerry_K 2023. 12. 6. 16:10

전에 1탄의 문제는

network에서 기록된 여러 개의 products 중

한 개의 products만 가져오는 것 였다.

이번 포스터에는 어떻게 한 페이지 안에 모든 찜을 가져오는 방법에 대해 소개하겠다.

(1탄을 안 본 사람이 있다면 꼭 보고 오세용... 아니면 이해하기 힘듬 !)

import requests

cookies = {
    'NNB': 'BZQEKFFD3ZGWG',
    'ASID': '75100b1e000001840cfb852400000053',
    'NV_WETR_LOCATION_RGN_M': '"MDUxMTAxMTg="',
    'NaverSuggestUse': 'unuse%26use',
    'autocomplete': 'use',
    'NV_WETR_LAST_ACCESS_RGN_M': '"MDUxMTAxMTg="',
    '_ga': 'GA1.1.687219659.1669681140',
    '_ga_K2ECMCJBFQ': 'GS1.1.1686294140.1.0.1686294145.0.0.0',
    '_ga_SQ24F7Q7YW': 'GS1.1.1686294140.1.0.1686294145.0.0.0',
    'SHP_BUCKET_ID': '9',
    'nx_ssl': '2',
    'nid_inf': '810134771',
    'NID_AUT': 'KzF8qxwc7Q16FWF2l9SHa44CCrK+w0d/hqsKCa7pUemktjJiOa1Y0wOJwiku023b',
    'NID_JKL': '2pCzNeGM0eRah+mdM4vS+Cgv12cmoVRaOHttCDzBnHY=',
    'ncpa': '95694|lprzjkoo|079082572a94ce3307d7ce5cfa1f98813527a107|95694|db6a4b03a9b918f109a3c9db4ab41365ce293096',
    'NID_SES': 'AAABfe+XwW4D1fcFvn9gTr9N4pEznpOL0OCRC6lsKknAC2fgdoyQ2D77YLHd8+A4fm9GgTKU1/tX9a5nyAhVtZszZHuf9RH2+y+RC6js8SHXrpdQ7+me0X/WFtOSQVw6bthv2mN+6Z3eDiiFueaifA538WRRN3yY7eeD8VR6+BfDfxN8UG+Ubu62gkahIaXH6NrYwiNNH9Fdq/fIiQ9jyx7XRh1SVWX5IhvioN8kROpWxKs46wMZwQWLbK9gxyrssjddKWbiI3A6M9ylMzmTFyeuNkZI5xN/RgZNeOwfSfu7q2K/T4AEaa0Kt/giJQ1kgeRyO5CRBXV07cUP5Nng6LYMkxbpxPGrmRmPFI2zSpt21B7RUk6d4dLEmJ0hBKdzn780Lr24rwzJgTs5F39oS5KibAnirlQr2mzi4ZdkKAC/rKY0nA5KIUADO7sWO3aHz7ZMkWQgc5UskJumYa5yshmnVH5xZLieEF8oMQ6fImX6EzfJsoHVzo7ffxK/b4cosQ3+pg==',
    'page_uid': 'iTJ6Bwqo15wssvxstkGssssssxl-399936',
    'spage_uid': 'iTJ6Bwqo15wssvxstkGssssssxl-399936',
}

headers = {
    'authority': 'search.shopping.naver.com',
    'accept': 'application/json, text/plain, */*',
    'accept-language': 'ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7',
    'content-type': 'application/json',
    # 'cookie': 'NNB=BZQEKFFD3ZGWG; ASID=75100b1e000001840cfb852400000053; NV_WETR_LOCATION_RGN_M="MDUxMTAxMTg="; NaverSuggestUse=unuse%26use; autocomplete=use; NV_WETR_LAST_ACCESS_RGN_M="MDUxMTAxMTg="; _ga=GA1.1.687219659.1669681140; _ga_K2ECMCJBFQ=GS1.1.1686294140.1.0.1686294145.0.0.0; _ga_SQ24F7Q7YW=GS1.1.1686294140.1.0.1686294145.0.0.0; SHP_BUCKET_ID=9; nx_ssl=2; nid_inf=810134771; NID_AUT=KzF8qxwc7Q16FWF2l9SHa44CCrK+w0d/hqsKCa7pUemktjJiOa1Y0wOJwiku023b; NID_JKL=2pCzNeGM0eRah+mdM4vS+Cgv12cmoVRaOHttCDzBnHY=; ncpa=95694|lprzjkoo|079082572a94ce3307d7ce5cfa1f98813527a107|95694|db6a4b03a9b918f109a3c9db4ab41365ce293096; NID_SES=AAABfe+XwW4D1fcFvn9gTr9N4pEznpOL0OCRC6lsKknAC2fgdoyQ2D77YLHd8+A4fm9GgTKU1/tX9a5nyAhVtZszZHuf9RH2+y+RC6js8SHXrpdQ7+me0X/WFtOSQVw6bthv2mN+6Z3eDiiFueaifA538WRRN3yY7eeD8VR6+BfDfxN8UG+Ubu62gkahIaXH6NrYwiNNH9Fdq/fIiQ9jyx7XRh1SVWX5IhvioN8kROpWxKs46wMZwQWLbK9gxyrssjddKWbiI3A6M9ylMzmTFyeuNkZI5xN/RgZNeOwfSfu7q2K/T4AEaa0Kt/giJQ1kgeRyO5CRBXV07cUP5Nng6LYMkxbpxPGrmRmPFI2zSpt21B7RUk6d4dLEmJ0hBKdzn780Lr24rwzJgTs5F39oS5KibAnirlQr2mzi4ZdkKAC/rKY0nA5KIUADO7sWO3aHz7ZMkWQgc5UskJumYa5yshmnVH5xZLieEF8oMQ6fImX6EzfJsoHVzo7ffxK/b4cosQ3+pg==; page_uid=iTJ6Bwqo15wssvxstkGssssssxl-399936; spage_uid=iTJ6Bwqo15wssvxstkGssssssxl-399936',
    'origin': 'https://search.shopping.naver.com',
    'referer': 'https://search.shopping.naver.com/search/all?adQuery=%ED%95%B8%EB%93%9C%ED%8F%B0%EC%BC%80%EC%9D%B4%EC%8A%A4&frm=NVSCPRO&origQuery=%ED%95%B8%EB%93%9C%ED%8F%B0%EC%BC%80%EC%9D%B4%EC%8A%A4&pagingIndex=2&pagingSize=40&productSet=total&query=%ED%95%B8%EB%93%9C%ED%8F%B0%EC%BC%80%EC%9D%B4%EC%8A%A4&sort=rel&timestamp=&viewType=list',
    'sbth': '0f5182f1eaeffb81e03a4287efca03680135c0777cf580e3456aad636af8b47097dd32dc7f0ee3a9c152fd02ebae6ce7',
    'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
    'sec-ch-ua-arch': '"x86"',
    'sec-ch-ua-bitness': '"64"',
    'sec-ch-ua-full-version-list': '"Google Chrome";v="119.0.6045.200", "Chromium";v="119.0.6045.200", "Not?A_Brand";v="24.0.0.0"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-model': '""',
    'sec-ch-ua-platform': '"Windows"',
    'sec-ch-ua-platform-version': '"10.0.0"',
    'sec-ch-ua-wow64': '?0',
    'sec-fetch-dest': 'empty',
    'sec-fetch-mode': 'cors',
    'sec-fetch-site': 'same-origin',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
}

json_data = {
    'params': [
        {
            'nvMid': '86788178899',
            'chpid': '9243678576',
            'isBook': False,
        },
        {
            'nvMid': '43154469618',
            'chpid': None,
            'isBook': False,
        },
        {
            'nvMid': '86722503211',
            'chpid': '9178002888',
            'isBook': False,
        },
        {
            'nvMid': '40229214618',
            'chpid': None,
            'isBook': False,
        },
        {
            'nvMid': '85175362168',
            'chpid': '7630861846',
            'isBook': False,
        },
        {
            'nvMid': '85056420009',
            'chpid': '7511919687',
            'isBook': False,
        },
        {
            'nvMid': '86564320302',
            'chpid': '9019819979',
            'isBook': False,
        },
        {
            'nvMid': '44160227618',
            'chpid': None,
            'isBook': False,
        },
        {
            'nvMid': '84089335517',
            'chpid': '6544835184',
            'isBook': False,
        },
        {
            'nvMid': '86787063641',
            'chpid': '9242563318',
            'isBook': False,
        },
        {
            'nvMid': '43030319626',
            'chpid': None,
            'isBook': False,
        },
        {
            'nvMid': '29904044618',
            'chpid': None,
            'isBook': False,
        },
        {
            'nvMid': '42647503618',
            'chpid': None,
            'isBook': False,
        },
        {
            'nvMid': '43877738621',
            'chpid': None,
            'isBook': False,
        },
        {
            'nvMid': '85470288646',
            'chpid': '7925788323',
            'isBook': False,
        },
        {
            'nvMid': '85248406363',
            'chpid': '7703906041',
            'isBook': False,
        },
        {
            'nvMid': '85585361259',
            'chpid': '8040860936',
            'isBook': False,
        },
        {
            'nvMid': '85176663865',
            'chpid': '7632163543',
            'isBook': False,
        },
        {
            'nvMid': '86935019199',
            'chpid': '9390518876',
            'isBook': False,
        },
        {
            'nvMid': '86316918545',
            'chpid': '8772418222',
            'isBook': False,
        },
    ],
}

response = requests.post(
    'https://search.shopping.naver.com/api/product-zzim/products',
    cookies=cookies,
    headers=headers,
    json=json_data,
)

# Note: json_data will not be serialized by requests
# exactly as it was in the original request.
#data = '{"params":[{"nvMid":"86788178899","chpid":"9243678576","isBook":false},{"nvMid":"43154469618","chpid":null,"isBook":false},{"nvMid":"86722503211","chpid":"9178002888","isBook":false},{"nvMid":"40229214618","chpid":null,"isBook":false},{"nvMid":"85175362168","chpid":"7630861846","isBook":false},{"nvMid":"85056420009","chpid":"7511919687","isBook":false},{"nvMid":"86564320302","chpid":"9019819979","isBook":false},{"nvMid":"44160227618","chpid":null,"isBook":false},{"nvMid":"84089335517","chpid":"6544835184","isBook":false},{"nvMid":"86787063641","chpid":"9242563318","isBook":false},{"nvMid":"43030319626","chpid":null,"isBook":false},{"nvMid":"29904044618","chpid":null,"isBook":false},{"nvMid":"42647503618","chpid":null,"isBook":false},{"nvMid":"43877738621","chpid":null,"isBook":false},{"nvMid":"85470288646","chpid":"7925788323","isBook":false},{"nvMid":"85248406363","chpid":"7703906041","isBook":false},{"nvMid":"85585361259","chpid":"8040860936","isBook":false},{"nvMid":"85176663865","chpid":"7632163543","isBook":false},{"nvMid":"86935019199","chpid":"9390518876","isBook":false},{"nvMid":"86316918545","chpid":"8772418222","isBook":false}]}'
#response = requests.post(
#    'https://search.shopping.naver.com/api/product-zzim/products',
#    cookies=cookies,
#    headers=headers,
#    data=data,
#)

이게 전에 products에서 zzim의 개수를 가져오는 카운터이다.

[목표 : 한 페이지 내 모든 찜 개수 카운트]

우선 cookies/headers는 왠만하면 건드리지 않는게 좋다.

그럼 누가봐도 가장 의심스러은 params, 즉 저 파라미터의 값을 변경해주면된다.

chpid는 뭔지 모르지만, 지우고 실행해도 문제없이 잘 출력이 되었으니,

가볍게 False로 개무시한다.

그럼 이제 우리는 한 페이지 내에 있는 모든 상품의 'nvMid' 만 알면된다.

네이버 쇼핑에 아무 키워드나 입력하고 네트워크창을 켜보자.

스크롤을 내리면 새로운 정보들이 서버로 부터 불려와진다.

네트워크 창에 xhr 타입의 products 밖에 안나온다.

근데 products에는 zzim 밖에 없다...

이럴때는 웹 페이지에 변화를 주면된다. (이거 잘 모르면 전에 포스트들 참고해보길 !!)

새로고침을 해보든 다른 페이지를 넘어가든 해보면 된다.

이것들을 하나하나 다 살펴보면된다.

꿀팁은 preview나 Response 위주로 보고 정보가 있으면 가져오면 되는 것이다.

(Payload는 어차피 curl(bash)로 가져오면 다 가져와져서 안봐도 됨)

다른 꿀팁으로는

네트워크창에 ctrl+f를 한 다음 내가 원하는 값을 검색하면 된다.

그러면 요로콤 나오는데 css,js 이런 잡다한 확장자는 보지만 않으면,

나름 쉽게 원하는 값을 가지고 있는 파일을 찾을 수 있다.

이렇게 하면 사실상 끝인거다 ...

내가 찾고자하는 'nvMid' 가 있다고 하니 한번 구체적을 확인해보자.

all?adQuery...~ - preview - shoppingResult - products - 2(아무거나) 의 순서대로 클릭해보자.

(참고로 여기에서 joson 형식의 딕셔너리로 표현되고 있는데, 이 구조들이 복잡해서 파이썬에서 파싱할 때 힘들다.

때문에 preview에서 구조 파악해서 파싱하면 쉽게 된다.)

그럼 이제 이렇게 뜨는데 문제는 내가 원하는 nvMid가 없다 ... (ㅂㄷㅂㄷ)

계속 찾아보니 id가 있었는데

혹시 몰라 검색해보니 products와 일치했다... (이렇게 다른 경우는 그냥 찾는수 밖에 ㅜㅜ)

휴우우우....

이제 다 끝났다 ㅠㅠ !!!

그럼 이제 products와 all?~~ 이 파일을 cmd(bash) 형태로 복사 한 후

파이썬 코드로 받으면 된다. (모르겠다면 전 포스팅 참고 !)

코드 작성

#products에 대한 curl 복사
product_cookies = {
    'NNB': '6TTK2RJU3JPWG',
    'autocomplete': 'use',
    '_ga': 'GA1.2.1198469368.1669035888',
    'NV_WETR_LOCATION_RGN_M': '"MDUxMTAxMTE="',
    'ASID': '6f5bbfa400000184fa368e5f00000067',
    'SHP_BUCKET_ID': '4',
    'NV_WETR_LAST_ACCESS_RGN_M': '"MDUxMTAxMTE="',
    'nx_ssl': '2',
    'nid_inf': '813506172',
    'NID_JKL': '7GO4JgsGQ6jSmPyQ+d0zZPEWBjZqV6VoPgsqFdP360s=',
    'NaverSuggestUse': 'unuse%26use',
    'NID_AUT': 'D2QOxk1i/qhN7mlsiYlMrQnnUV8Gl+BXV86T78q51HMwExxzB7wcbgptaPWfOS0c',
    'ncpa': '494450|lpnzu3bc|33955d23247452d9f39b7befe57b1669f78a6422|s_223323393867677778|3739649221291330a79e03aba028f76e9b0939de:24|lpnzuyy8|eb0ed5873c0c6ebf2a6c5e362436a1ebf9eff7ce|s_419afe53a6bb|660b387db242b413e3d4d88cb426c264aa22cfb9:6506854|lpo0e5jc|f21ed58f1c6723c8cc4d00e3daa6a6422ffd0e32|s_202c49de12296|2945c3a090c4ef2300d5fd58af579649a7525b62:95694|lpo0g8ds|c5ac73032044e7c458f77e7f92f4cd0b391f5b71|95694|d5513b891cf3d61c63fc90b527f73eb7a2b1c655:367128|lpo0gbgw|c446229bc943cee174914f4c257713d2f205b404|s_172552030016403205|0ac152849d508c0f673b0dec0db0a8a25afb2d4b:1142874|lppah9xk|8c0bfc5cedba068684ebffb0372768a683b043e6|s_318df10bd1f2f|ddccaf77e58e8ff795e6da600b9730041ce0658a:1108389|lppbaieg|1a3ee417de104971a531073f1f19178ffd6a597e|s_2c6ae6bfc1750|78a17ad54d224316e33bd605f088b6eae27a49c0:8283684|lppgh5jc|da7cafc31641a6efb95608351d1829e697fd88f1|s_14ee9dbd76e3e|986bc50b91619ceced2d29619a58f16ee8584d8f:6|lpqs90ns|192ba3c47b3ad2ef4e536951fecef77a6e9dd7ec|s_265212fbc481|84b9a4af11a946f64d5d333ef74424f4381980c8:747614|lpr0n5z4|b441c92d23fe22bc3e5e5cc84b888c90f4f9b053|s_5054ed8eda8|1de7a41575f33ab35876a2dfe04960eee59b5be8',
    'NID_SES': 'AAABhdehcx1hqCDRwKi7RdEZJlcP+U7WTYoPGBhuju5otOuAM/7d1mcuKnPYtzbdhGi8AUef0e8oTIFcRDoo6voBNQwKA5HqDkW6ao49f3657oH0UHUGFDEDBzd7Yk1z7TjbbQaNV//pHXOaGA5ohVw1HQ8cs77WSftqxNN+r4mlcY7O34NsFj+Hzft9xUJ6JowN6Vjjpv221REzF0hVyz80+X5X3pmIQVEZLiSZUPLeziizML4RnHvq/xN10PKgXGtvSPHzH6+NMQtCd8lb2rlHmOL3YH1r3cjwLQdLF7ACl3NA5juj/NSXMZQazzIVh72MRH4xtLXjJ6J2vxlRSM+dX5Tlw1Q5+3S1Uoe40rvts1LbE+JmctBY2hxFhgg9NkDx8/fe4IPqzKllPxxbieYmQpm8y/VMU8kG8M/q9qRE6HtB5OSqp+r/up1VWuXzG6Sh7YWwq7lB9tjwVYcahdJLfGBWLrTXJP089SDvWLQAv+ohZ5OrVV39MKQLqUDFbd8AcIZrUB618MPpuJqc8Q8Ow5Q=',
    '_naver_usersession_': 's9o7YfH136DxWRX5MFx0CtsN',
    'page_uid': 'iTHdylqo1awssEayShGssssssao-112218',
    'NX_BT_ID': '1',
    'spage_uid': 'iTHdylqo1awssEayShGssssssao-112218',
}

product_headers = {
    'authority': 'search.shopping.naver.com',
    'accept': 'application/json, text/plain, */*',
    'accept-language': 'ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7',
    # 'cookie': 'NNB=6TTK2RJU3JPWG; autocomplete=use; _ga=GA1.2.1198469368.1669035888; NV_WETR_LOCATION_RGN_M="MDUxMTAxMTE="; ASID=6f5bbfa400000184fa368e5f00000067; SHP_BUCKET_ID=4; NV_WETR_LAST_ACCESS_RGN_M="MDUxMTAxMTE="; nx_ssl=2; nid_inf=813506172; NID_JKL=7GO4JgsGQ6jSmPyQ+d0zZPEWBjZqV6VoPgsqFdP360s=; NaverSuggestUse=unuse%26use; NID_AUT=D2QOxk1i/qhN7mlsiYlMrQnnUV8Gl+BXV86T78q51HMwExxzB7wcbgptaPWfOS0c; ncpa=494450|lpnzu3bc|33955d23247452d9f39b7befe57b1669f78a6422|s_223323393867677778|3739649221291330a79e03aba028f76e9b0939de:24|lpnzuyy8|eb0ed5873c0c6ebf2a6c5e362436a1ebf9eff7ce|s_419afe53a6bb|660b387db242b413e3d4d88cb426c264aa22cfb9:6506854|lpo0e5jc|f21ed58f1c6723c8cc4d00e3daa6a6422ffd0e32|s_202c49de12296|2945c3a090c4ef2300d5fd58af579649a7525b62:95694|lpo0g8ds|c5ac73032044e7c458f77e7f92f4cd0b391f5b71|95694|d5513b891cf3d61c63fc90b527f73eb7a2b1c655:367128|lpo0gbgw|c446229bc943cee174914f4c257713d2f205b404|s_172552030016403205|0ac152849d508c0f673b0dec0db0a8a25afb2d4b:1142874|lppah9xk|8c0bfc5cedba068684ebffb0372768a683b043e6|s_318df10bd1f2f|ddccaf77e58e8ff795e6da600b9730041ce0658a:1108389|lppbaieg|1a3ee417de104971a531073f1f19178ffd6a597e|s_2c6ae6bfc1750|78a17ad54d224316e33bd605f088b6eae27a49c0:8283684|lppgh5jc|da7cafc31641a6efb95608351d1829e697fd88f1|s_14ee9dbd76e3e|986bc50b91619ceced2d29619a58f16ee8584d8f:6|lpqs90ns|192ba3c47b3ad2ef4e536951fecef77a6e9dd7ec|s_265212fbc481|84b9a4af11a946f64d5d333ef74424f4381980c8:747614|lpr0n5z4|b441c92d23fe22bc3e5e5cc84b888c90f4f9b053|s_5054ed8eda8|1de7a41575f33ab35876a2dfe04960eee59b5be8; NID_SES=AAABhdehcx1hqCDRwKi7RdEZJlcP+U7WTYoPGBhuju5otOuAM/7d1mcuKnPYtzbdhGi8AUef0e8oTIFcRDoo6voBNQwKA5HqDkW6ao49f3657oH0UHUGFDEDBzd7Yk1z7TjbbQaNV//pHXOaGA5ohVw1HQ8cs77WSftqxNN+r4mlcY7O34NsFj+Hzft9xUJ6JowN6Vjjpv221REzF0hVyz80+X5X3pmIQVEZLiSZUPLeziizML4RnHvq/xN10PKgXGtvSPHzH6+NMQtCd8lb2rlHmOL3YH1r3cjwLQdLF7ACl3NA5juj/NSXMZQazzIVh72MRH4xtLXjJ6J2vxlRSM+dX5Tlw1Q5+3S1Uoe40rvts1LbE+JmctBY2hxFhgg9NkDx8/fe4IPqzKllPxxbieYmQpm8y/VMU8kG8M/q9qRE6HtB5OSqp+r/up1VWuXzG6Sh7YWwq7lB9tjwVYcahdJLfGBWLrTXJP089SDvWLQAv+ohZ5OrVV39MKQLqUDFbd8AcIZrUB618MPpuJqc8Q8Ow5Q=; _naver_usersession_=s9o7YfH136DxWRX5MFx0CtsN; page_uid=iTHdylqo1awssEayShGssssssao-112218; NX_BT_ID=1; spage_uid=iTHdylqo1awssEayShGssssssao-112218',
    'logic': 'PART',
    'referer': 'https://search.shopping.naver.com/search/all?query=%ED%95%B8%EB%93%9C%ED%8F%B0+%EC%BC%80%EC%9D%B4%EC%8A%A4&bt=1&frm=NVSCPRO',
    'sbth': 'be47be4da24bb869d213e377404c25cbb8971073c85d515da365bbb2b9be9b8a31b330aab53ed4a06fecd56859715715',
    'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
    'sec-ch-ua-arch': '"x86"',
    'sec-ch-ua-bitness': '"64"',
    'sec-ch-ua-full-version-list': '"Google Chrome";v="119.0.6045.200", "Chromium";v="119.0.6045.200", "Not?A_Brand";v="24.0.0.0"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-model': '""',
    'sec-ch-ua-platform': '"Windows"',
    'sec-ch-ua-platform-version': '"15.0.0"',
    'sec-ch-ua-wow64': '?0',
    'sec-fetch-dest': 'empty',
    'sec-fetch-mode': 'cors',
    'sec-fetch-site': 'same-origin',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
}

#all?~~ 에 대한 curl 복사

zzim_cookies = {
        'NNB': '6TTK2RJU3JPWG',
        'autocomplete': 'use',
        '_ga': 'GA1.2.1198469368.1669035888',
        'NV_WETR_LOCATION_RGN_M': '"MDUxMTAxMTE="',
        'ASID': '6f5bbfa400000184fa368e5f00000067',
        'SHP_BUCKET_ID': '4',
        'NV_WETR_LAST_ACCESS_RGN_M': '"MDUxMTAxMTE="',
        'nx_ssl': '2',
        'nid_inf': '813506172',
        'NID_AUT': '6kiAbtWvFZK3y90sHs9cfKS12yY1zkUBJdBmlEKIM2kq2fWAmKgi9OnXOnz9nUyL',
        'NID_JKL': '7GO4JgsGQ6jSmPyQ+d0zZPEWBjZqV6VoPgsqFdP360s=',
        'page_uid': 'iTrAvdqVOZCssRoVy7Vssssstcl-188131',
        'spage_uid': 'iTrAvdqVOZCssRoVy7Vssssstcl-188131',
        'NID_SES': 'AAABhRMGcG38dd+AVbQV0majyYVy//mj0pMGIdwEgF50kuXMG7yzS7NgEWATTyWEDz+h3igfc8lmNMdqxPT3hNZiIXQyGAPpPlJaomIY1eyU8eMG1nMOIQDWyQqZUkNapJKyXwt1ZsBZYkQlba3no2oFokuyLoL/TWn8p2i7tGNYdeKCT6J/RW0prfsCZuDRlIR+aXy0s48Wp6jB0j9xQM9Swzt2BxD2EBPvUvZJVAqyQKrjoPYsxhBo0mwd0C3L2oxnQUPG1bx2IlAs8dFEtoAz17Kx6YxF2b0XPVs8ffK8ZGgHs6192PTpVLk2IVtSKa5BFZN6Q7fgBePj/WZ0XCLp8s2BxGw2qdl3IXeezVA3f3Xl46B2cuZQPeEUKrZUS7tIFdbwiDR8LPrue/e/EK2ZZsQIzyayOuzeEXJYshF0ZD2UOGULTWv2NhkLVeLKqd1hq3oBPxX/wBkQjeKBqeJde8OjOG980cj+qRVCbCsmedoqrwP+yLta99Ksmt7res+DArlvf8PLzrHVb7ZYrCcFF0E=',
        'ncpa': '8906154|lpl7awf4|42d1baece663e0d213ba404621cf481b7242f35f|s_e26988f9dc99|6c2d64a4c1df76e2280336de7f219a7a355468a1:182410|lpl8m8w8|15c34379acb36244a25e6884b137d9e6e5db97d7|s_1eeb12ba8dfe6bdf|a5e20b94bc6a7baaa057cd3c57fea5e0848efe86:24|lpmnyhxk|fa6bbca73f1d69f3e74162fd22aa9cd128b177b2|s_419afe53a6bb|7cb0f74e460fe247033a56e43cca0af5be4fb347:95694|lpnwg4vs|6bc56e225781b65431a50241c5ef46e62183461b|95694|c9653a506f034c066b2b000523448796fc82c83c',
        'NaverSuggestUse': 'unuse%26use',
    }

zzim_headers = {
    'authority': 'search.shopping.naver.com',
    'accept': 'application/json, text/plain, */*',
    'accept-language': 'ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7',
    'content-type': 'application/json',
    # 'cookie': 'NNB=6TTK2RJU3JPWG; autocomplete=use; _ga=GA1.2.1198469368.1669035888; NV_WETR_LOCATION_RGN_M="MDUxMTAxMTE="; ASID=6f5bbfa400000184fa368e5f00000067; SHP_BUCKET_ID=4; NV_WETR_LAST_ACCESS_RGN_M="MDUxMTAxMTE="; nx_ssl=2; nid_inf=813506172; NID_AUT=6kiAbtWvFZK3y90sHs9cfKS12yY1zkUBJdBmlEKIM2kq2fWAmKgi9OnXOnz9nUyL; NID_JKL=7GO4JgsGQ6jSmPyQ+d0zZPEWBjZqV6VoPgsqFdP360s=; page_uid=iTrAvdqVOZCssRoVy7Vssssstcl-188131; spage_uid=iTrAvdqVOZCssRoVy7Vssssstcl-188131; NID_SES=AAABhRMGcG38dd+AVbQV0majyYVy//mj0pMGIdwEgF50kuXMG7yzS7NgEWATTyWEDz+h3igfc8lmNMdqxPT3hNZiIXQyGAPpPlJaomIY1eyU8eMG1nMOIQDWyQqZUkNapJKyXwt1ZsBZYkQlba3no2oFokuyLoL/TWn8p2i7tGNYdeKCT6J/RW0prfsCZuDRlIR+aXy0s48Wp6jB0j9xQM9Swzt2BxD2EBPvUvZJVAqyQKrjoPYsxhBo0mwd0C3L2oxnQUPG1bx2IlAs8dFEtoAz17Kx6YxF2b0XPVs8ffK8ZGgHs6192PTpVLk2IVtSKa5BFZN6Q7fgBePj/WZ0XCLp8s2BxGw2qdl3IXeezVA3f3Xl46B2cuZQPeEUKrZUS7tIFdbwiDR8LPrue/e/EK2ZZsQIzyayOuzeEXJYshF0ZD2UOGULTWv2NhkLVeLKqd1hq3oBPxX/wBkQjeKBqeJde8OjOG980cj+qRVCbCsmedoqrwP+yLta99Ksmt7res+DArlvf8PLzrHVb7ZYrCcFF0E=; ncpa=8906154|lpl7awf4|42d1baece663e0d213ba404621cf481b7242f35f|s_e26988f9dc99|6c2d64a4c1df76e2280336de7f219a7a355468a1:182410|lpl8m8w8|15c34379acb36244a25e6884b137d9e6e5db97d7|s_1eeb12ba8dfe6bdf|a5e20b94bc6a7baaa057cd3c57fea5e0848efe86:24|lpmnyhxk|fa6bbca73f1d69f3e74162fd22aa9cd128b177b2|s_419afe53a6bb|7cb0f74e460fe247033a56e43cca0af5be4fb347:95694|lpnwg4vs|6bc56e225781b65431a50241c5ef46e62183461b|95694|c9653a506f034c066b2b000523448796fc82c83c; NaverSuggestUse=unuse%26use',
    'origin': 'https://search.shopping.naver.com',
    'referer': 'https://search.shopping.naver.com/search/all?query=%EC%95%A1%EC%A0%95%ED%95%84%EB%A6%84&cat_id=&frm=NVSHATC',
    'sbth': '77074ab4693c2007419f7fba27a0d1de44c43a4f3d9d2d265814254b5a74210643c3a3a92780474e1ddc1b64fdf8a0fd',
    'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
    'sec-ch-ua-arch': '"x86"',
    'sec-ch-ua-bitness': '"64"',
    'sec-ch-ua-full-version-list': '"Google Chrome";v="119.0.6045.200", "Chromium";v="119.0.6045.200", "Not?A_Brand";v="24.0.0.0"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-model': '""',
    'sec-ch-ua-platform': '"Windows"',
    'sec-ch-ua-platform-version': '"15.0.0"',
    'sec-ch-ua-wow64': '?0',
    'sec-fetch-dest': 'empty',
    'sec-fetch-mode': 'cors',
    'sec-fetch-site': 'same-origin',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
}

keyword  = "핸드폰"
product_params = {
        'adQuery': f'{keyword}',
        'eq': '',
        'frm': 'NVSHATC',
        'iq': '',
        'origQuery': f'{keyword}',
        'pagingIndex': '1',
        'pagingSize': '40',
        'productSet': 'total',
        'query': f'{keyword}',
        'sort': 'rel',
        'viewType': 'list',
        'xq': '',
    }

response = requests.get('https://search.shopping.naver.com/api/search/all', params=product_params, cookies=product_cookies, headers=product_headers)
itemlists = json.loads(response.text)

for idx,product in enumerate(itemlists["shoppingResult"]["products"]):

    id = product["id"]

    # Zzim 개수 카운트
    json_data = {
        'params': [
        {
            'nvMid': f'{id}',
            'chpid': None,
            'isBook': False,
            },
        ]
    }

    response = requests.post(
        'https://search.shopping.naver.com/api/product-zzim/products',
        cookies=zzim_cookies,
        headers=zzim_headers,
        json=json_data,
    )
    zzim_json = json.loads(response.text)

    print(f'{idx+1}. {id}(id별 찜 개수) : {zzim_json["zzim"][id]["count"]}')

이렇게 하면 내가 원하는 것들을 잘 크롤링 할 수 있다.

세상에 requests로 안되는 크롤링은 없다 !!!??

network를 잘 보고 변화만 잘봐도 왠만한 크롤링은 다 하는 것 같다.

마지막으로 여기에서 더 자세한 내용은

javascript로 되어있었는데,

꼭 network에만 모든 정보가 있는게 아닌걸 알아두자 !

'데이터 > 크롤링' 카테고리의 다른 글

[Python] 네이버 스토어 찜 크롤링 with requests _1탄 (2)	2023.12.05
네이버 크롤링 연결 제한 푸는법 (1)	2023.11.30
[Python] requests로 네이버 데이터랩 크롤링 (0)	2023.11.10
[Python] selenium 없이 스크롤 하는 법 (네이버쇼핑) (1)	2023.10.15

현재글[Python] 네이버 스토어 찜 크롤링 with requests _2탄

Jerry의 컴퓨터 일기 💻

connecting the dots

Today :
Yesterday :

Jerry의 컴퓨터 일기 💻