f

tieba/config.yaml

@@ -2,14 +2,16 @@ debug: 1
 dev:
   db: root:mH9TaSz#FV43T*z%@(111.231.204.74:3306)/mepai_v2
   qcloud:
-    region: sh
+    region: shanghai
     bucket: devmepai
     secret-id: AKIDNcdPozQhJSXWXNtUinnT1OmwqhhT6NQ0
     secret-key: TgdNr89wc2OUK5aomX1IJ8lCVMN82n0y
+    app-id: 1251461203
 prod:
-  db:
+  db: cdb_outer_mepai:w9qUd%=Zj&Th@(sh-cdb-reajbfk8.sql.tencentcdb.com:62820)/mepainew
   qcloud:
-    region:
-    bucket:
-    secret-id:
-    secret-key:
\ No newline at end of file
+    region: shanghai
+    bucket: mepainew
+    secret-id: AKIDNcdPozQhJSXWXNtUinnT1OmwqhhT6NQ0
+    secret-key: TgdNr89wc2OUK5aomX1IJ8lCVMN82n0y
+    app-id: 1251461203
\ No newline at end of file

tieba/main.go

@@ -16,6 +16,8 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
+	os.RemoveAll("./avatars")
+	os.Mkdir("./avatars", 0777)
 	service.SetConfig(cstr)
 	service.Open(os.Args[1])
 	service.Begin();

tieba/service/service.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/PuerkitoBio/goquery"
 	"github.com/chromedp/cdproto/cdp"
 	"github.com/chromedp/cdproto/network"
 	"github.com/chromedp/cdproto/runtime"
@@ -13,6 +14,7 @@ import (
 	"math/rand"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -36,7 +38,6 @@ func SaveUser(users []User) {
 			u.Avatar = "https:" + u.Avatar
 		}
 		if err != nil {
-			fmt.Printf("无法访问:%s", u.Avatar)
 			continue
 		}
 		defer res.Body.Close()
@@ -48,7 +49,7 @@ func SaveUser(users []User) {
 
 func Open(url string) {
 	options := []chromedp.ExecAllocatorOption{
-		chromedp.Flag("headless",true),
+		chromedp.Flag("headless",false),
 		chromedp.Flag("blink-settings","imageEnable=false"),
 		chromedp.UserAgent(`Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)`),
 	}
@@ -58,12 +59,30 @@ func Open(url string) {
 	timeOutCtx, cancel := context.WithTimeout(chromeCtx, 6000* time.Second)
 	defer cancel()
 	var users  []User
+	var nextId  int
+	var qr string
+	var h string
+	var hasNext int
 	chromedp.Run(chromeCtx, setCookies(getCookies()))
 	chromedp.Run(timeOutCtx,
 		chromedp.Navigate(url),
 		chromedp.ActionFunc(func(ctx context.Context) error {
 Loop:
-			time.Sleep(time.Duration(rand.Intn(10))* time.Second)
+			chromedp.Sleep(time.Second * time.Duration(rand.Intn(5)))
+			h = ""
+			nextId = 0
+			hasNext = 0
+			chromedp.OuterHTML("#thread_theme_5 > div.l_thread_info > ul",&h).Do(ctx)
+			if h != "" {
+				doc, _ := goquery.NewDocumentFromReader(strings.NewReader(h))
+				doc.Find("li").Children().Each(func(i int, selection *goquery.Selection) {
+					if selection.Text() == "下一页" {
+						nextId = i
+						hasNext = hasNext + 1;
+					}
+				})
+				qr = strconv.Itoa(nextId + 1)
+			}
 			var nodes []*cdp.Node
 			users = users[0:0]
 			chromedp.Nodes("#j_p_postlist > div  > div.d_author > ul > li.icon > div > a > img", &nodes).Do(ctx)
@@ -73,21 +92,22 @@ Loop:
 				users = append(users, User{
 					Avatar: src,
 					Nickname: username,
-					Gender: -1,
 				})
 			}
 			SaveUser(users)
-			chromedp.ScrollIntoView("#tb_rich_poster > div.poster_head.clearfix > div.poster_head_text > a").Do(ctx)
-			goLoop := chromedp.WaitVisible("#thread_theme_7 > div.l_thread_info > ul > li.l_pager.pager_theme_5.pb_list_pager > a:nth-child(7)",
+			if hasNext <= 0{
+				return nil
+			}
+			goLoop := chromedp.WaitVisible("#thread_theme_5 > div.l_thread_info > ul > li.l_pager.pager_theme_4.pb_list_pager > a:nth-child("+ qr +")",
 				chromedp.ByQuery, chromedp.AtLeast(0), chromedp.After(func(ctx2 context.Context, id runtime.ExecutionContextID, node ...*cdp.Node) error {
 					if len(node) <= 0 {
 						return nil
 					}
-					chromedp.Click("#thread_theme_7 > div.l_thread_info > ul > li.l_pager.pager_theme_5.pb_list_pager > a:nth-child(7)").Do(ctx2)
+					chromedp.Click("#thread_theme_5 > div.l_thread_info > ul > li.l_pager.pager_theme_4.pb_list_pager > a:nth-child("+ qr +")").Do(ctx2)
 					return errors.New("go");
 				}),
 			).Do(ctx)
-			if goLoop != nil {
+			if goLoop != nil && goLoop.Error() == "go" {
 				goto Loop
 			}
 			return nil
@@ -96,6 +116,15 @@ Loop:
 
 }
 
+func getTotalPage(s string) int  {
+	if s == "" {
+		return 1
+	}
+	d,_ := goquery.NewDocumentFromReader(strings.NewReader(s))
+	p,_ := strconv.Atoi(d.Text())
+	return p
+}
+
 func getCookies()  map[string]string {
 	file,err := ioutil.ReadFile("./data/cookie.txt")
 	if err != nil {

tieba/service/update.go

@@ -3,6 +3,7 @@ package service
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/go-xorm/xorm"
@@ -31,8 +32,9 @@ type Detail struct {
 type QCloud struct {
 	Region string
 	Bucket string
-	SecretId string
-	SecretKey string
+	SecretId string `yaml:"secret-id"`
+	SecretKey string `yaml:"secret-key"`
+	AppId string `yaml:"app-id"`
 }
 
 
@@ -49,12 +51,6 @@ type UserModel struct {
 	NicknameLower string `json:"nickname_lower"`
 }
 
-type CosConfig struct {
-	SecretId string 	`yaml:"secret-id"`
-	SecretKey string `yaml:"secret-key"`
-	Bucket string
-	Region string
-}
 
 var dbc *xorm.Engine
 var config Detail
@@ -78,8 +74,8 @@ func SetConfig(cstr []byte) {
 	dbc.ShowSQL(true)
 }
 
-func upload(stream []byte, remotePath string) {
-	u, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", config.QCloud.Bucket, config.QCloud.Region))
+func Upload(stream []byte, remotePath string) {
+	u, _ := url.Parse(fmt.Sprintf("https://%s-%s.cos.ap-%s.myqcloud.com", config.QCloud.Bucket, config.QCloud.AppId, config.QCloud.Region))
 	b := &cos.BaseURL{BucketURL: u}
 	c := cos.NewClient(b, &http.Client{
 		Timeout: 100 * time.Second,
@@ -89,7 +85,14 @@ func upload(stream []byte, remotePath string) {
 		},
 	})
 	ctx := context.Background()
-	c.Object.Put(ctx, remotePath, bytes.NewReader(stream), nil)
+	r,err := c.Object.Put(ctx, remotePath, bytes.NewReader(stream), nil)
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+	defer r.Body.Close()
+	res,_ := ioutil.ReadAll(r.Body)
+	fmt.Println(string(res))
 }
 
 
@@ -101,7 +104,7 @@ func Begin() {
 		items = items[0:0]
 		err := dbc.Table("mepai_fuser").
 			Join("LEFT", "mepai_user", "mepai_fuser.uid=mepai_user.id").
-			Where("mepai_user.nickname_lower like ?", "用户%").
+			Where("mepai_user.nickname_lower like ?", "%用户%").
 			And("mepai_fuser.id > ?", lastId).
 			Asc("mepai_fuser.id").
 			Limit(30).
@@ -113,19 +116,36 @@ func Begin() {
 			break
 		}
 		lastId = items[len(items) - 1].Id
-		updateUser(items)
+		err = updateUser(items)
+		if err != nil {
+			fmt.Println(err.Error())
+			break;
+		}
 	}
 }
 
-func updateUser(users []FUser) {
+func updateUser(users []FUser)  error{
 	for _,u := range users {
-		info := getOne(u.Uid);
+		info, nook := getOne(u.Uid);
+		if nook != nil {
+			return nook
+		}
 		fmt.Printf("获取到%s给用户%d,头像:%s\n", info.Nickname, u.Uid, info.Avatar)
-		_,err := dbc.Table("mepai_user").Cols("nickname", "avatar", "nickname_lower").Update(info)
+		_,err := dbc.Table("mepai_user").
+			Cols("nickname", "avatar", "nickname_lower").
+			Update(info, map[string]interface{} {
+			"id": u.Uid,
+		})
+		dbc.Table("mepai_fuser").Cols("status").Update(map[string]interface{} {
+			"status": 1,
+		}, map[string]interface{} {
+			"uid": u.Uid,
+		})
 		if err != nil {
-			fmt.Println(err.Error())
+			return err
 		}
 	}
+	return nil
 }
 
 
@@ -141,13 +161,17 @@ func ReadAll() {
 			allInfo = append(allInfo, fi.Name())
 		}
 	}
+	fmt.Printf("获取到总计图片:%d张", len(allInfo))
 }
 
-func getOne(uid int) UserModel {
-	var one string
-	var user UserModel
+func getOne(uid int) (UserModel, error) {
 	dir := "./avatars/"
 Loop:
+	if len(allInfo) <= 0 {
+		return UserModel{}, errors.New("not enough")
+	}
+	var one string
+	var user UserModel
 	one, allInfo = allInfo[0], allInfo[1:]
 	_,err := dbc.Table("mepai_user").Where("nickname_lower = ?", one).Get(&user)
 	if err != nil {
@@ -159,13 +183,13 @@ Loop:
 	path := fmt.Sprintf("/app/u/%d/avatar_%s.jpg", uid, CreateRandomString(16))
 	filePath := dir + one
 	stream,_ := ioutil.ReadFile(filePath)
-	upload(stream, path)
+	Upload(stream, path)
 	user.Id = uid
 	user.Avatar = path
 	user.Nickname = one
 	user.NicknameLower = one
 	os.Remove(filePath)
-	return user
+	return user, nil
 }
 
 func CreateRandomString(len int) string  {

tieba/test/s_test.go

@@ -3,7 +3,10 @@ package test
 import (
 	"bytes"
 	"fmt"
+	"io/ioutil"
 	"math/big"
+	"mepai_t/service"
+	"os"
 	"testing"
 	"crypto/rand"
 )
@@ -23,4 +26,14 @@ func CreateRandomString(len int) string  {
 		container += string(str[randomInt.Int64()])
 	}
 	return container
-}
\ No newline at end of file
+}
+
+func TestUpload(t *testing.T)  {
+	cstr,_ := os.ReadFile("../config.yaml")
+	service.SetConfig(cstr)
+	path := "/home/code001/Pictures/63ca17950a7b0208ded1503b6fd9f2d3562cc818.jpg"
+	stream,_ := ioutil.ReadFile(path)
+	remote := "/app/u/898/uplaodPic.jpg"
+	service.Upload(stream, remote);
+}
+