Mercurial > pumpbridge > pumpbridge
changeset 22:b90e6df48d2d
eachLimit instead of each
| author | Mathias Gebbe <mgebbe@intevation.de> |
|---|---|
| date | Mon, 16 Jun 2014 17:31:05 +0200 |
| parents | dbe2232af20f |
| children | 8c56fdbac0fb |
| files | src/facebook.coffee src/fromESN.coffee src/google.coffee src/syncALONE.coffee src/toESN.coffee src/twitter.coffee src/usermap.coffee |
| diffstat | 7 files changed, 46 insertions(+), 82 deletions(-) [+] |
line wrap: on
line diff
--- a/src/facebook.coffee Fri Jun 13 21:21:41 2014 +0200 +++ b/src/facebook.coffee Mon Jun 16 17:31:05 2014 +0200 @@ -48,10 +48,10 @@ ], (err, posts) -> return if not posts? or posts.length is 0 #console.log JSON.stringify(posts) # all posts from network (100) - async.eachSeries posts, ((post, callback) -> + async.eachLimit posts, 10, ((post, callback) -> async.waterfall [ (callback) -> - FromESN.search {uid: post.id + "@facebook_to_" + me, recipientUser: me}, callback + FromESN.search {uid: post.id + "@facebook_to_" + me}, callback (result, callback) -> return if result.length isnt 0 or post.from.id is id #console.log "postid: " + post.from.id + "id:" + id @@ -87,6 +87,7 @@ else console.log 'all posts processed' return + #### # TO DO: GET NEW COMMENTS # for each fromESN check the comments @@ -98,7 +99,7 @@ (callback) -> FromESN.search {recipientUser: me}, callback (allESN, callback) -> - async.each allESN, ((fromesn, callback) -> + async.eachLimit allESN, 10, ((fromesn, callback) -> async.waterfall [ (cb) -> if fromesn.sourcePost.indexOf('www.facebook.com') is -1 @@ -129,7 +130,6 @@ ], (err, result) -> #done - # GET PUBLIC PUMP POSTS AND POST THEM async.waterfall [ (callback) ->
--- a/src/fromESN.coffee Fri Jun 13 21:21:41 2014 +0200 +++ b/src/fromESN.coffee Mon Jun 16 17:31:05 2014 +0200 @@ -22,6 +22,7 @@ "updated" ] indices: [ + "uid" "postid" "sourceUser" "sourcePost"
--- a/src/google.coffee Fri Jun 13 21:21:41 2014 +0200 +++ b/src/google.coffee Mon Jun 16 17:31:05 2014 +0200 @@ -44,16 +44,16 @@ getFriends(user,callback) ], (err, friends) -> # für jeden Freunde tue dies: hier bewusst von async.eachSeries friends, ((friend, callback) -> auf async.each gewechselt - async.each friends, ((friend, callback) -> + async.eachLimit friends, 10, ((friend, callback) -> async.waterfall [ (callback) -> getStream(user, friend, callback) (stream, callback) -> return if not stream? or not stream.items? or stream.items.length is 0 - async.each stream.items, ((post, cb) -> + async.eachLimit stream.items, 10, ((post, cb) -> async.waterfall [ (callback) -> - FromESN.search {uid: post.id + "@google_to_" + me, recipientUser: me}, callback + FromESN.search {uid: post.id + "@google_to_" + me}, callback (result, callback) -> return if result.length isnt 0 ti = new Date().getTime() - interval @@ -154,7 +154,7 @@ callback null,users (users, callback) -> #EdgeControl.removeEdges(me,'@google') - async.each users.items, ((user,cb) -> + async.eachLimit 10, users.items, ((user,cb) -> if user.id? friends.push(user.id) #Routes.updateUserDB(user.id+'@google',user.displayName,user.displayName,user.url,user.image.url)
--- a/src/syncALONE.coffee Fri Jun 13 21:21:41 2014 +0200 +++ b/src/syncALONE.coffee Mon Jun 16 17:31:05 2014 +0200 @@ -40,7 +40,8 @@ Usermap.scan ((user) -> if user.id.indexOf('@twitter') isnt -1 console.log "start sync for twitter user" - Twitter.sync(user) + Twitter.syncToESN(user) + Twitter.syncFromESN(user) ), (err) -> catch err console.log 'Error!' + err @@ -58,65 +59,6 @@ return -postParser = (post, user, network, callback) -> - parsed = "" - #console.log "\n" + post.id + "\n" + user.id + user.displayName - #PROFILE_LINK PROFILE_NAME PROFILE_PIC_LINK_80x80=$3 POST_LINK POST_TIME CONTENT - #text='<img src='$PROFILE_PIC_LINK_80x80'></img> <a href='$PROFILE_LINK'>'$PROFILE_NAME'</a> <a href='$POST_LINK'>wrotes</a> at '$POST_TIME':<br><br>'$CONTENT'' - - if network is 'facebook' - if user? and user.profilePicLink? - profilePicLink = user.profilePicLink.replace("50x50", "80x80") - else - profilePicLink = 'http://upload.wikimedia.org/wikipedia/commons/thumb/c/c2/F_icon.svg/80px-F_icon.svg.png' - - if post.type is 'status' - parsed = "<img src='" + profilePicLink + "'></img> <a href='" + user.profileLink + "'>" + user.displayName + "</a> <a href='https://www.facebook.com/" + post.id + "'> wrotes</a> via " + network + " at " + post.updated_time + ":<br><br>" + post.message - - if post.type is 'photo' - parsed = "<img src='" + profilePicLink + "'></img> <a href='" + user.profileLink + "'>" + user.displayName + "</a> <a href='"+ post.link + "'> wrotes</a> via " + network + " at " + post.updated_time + ":<br>" - parsed += "<br>" + post.message if post.message? and post.message isnt "" - parsed += "<br><img src='" + post.picture + "'></img>" - - if post.type is 'link' or post.type is 'video' - parsed = "<img src='" + profilePicLink + "'></img> <a href='" + user.profileLink + "'>" + user.displayName + "</a> <a href='https://www.facebook.com/" + post.id + "'> wrotes</a> via " + network + " at " + post.updated_time + ":<br>" - parsed += "<br>" + post.message if post.message? and post.message isnt "" - parsed += "<br>" + post.description if post.description? - parsed += "<br><a href='" + post.link + "'>"+ post.link + "</a>" if post.link? - parsed += "<br><img src='" + post.picture + "'></img>" if post.picture? - - callback null, parsed - - if network is 'google' - parsed = "<img src='" + post.actor.image.url + "'></img> <a href='" + post.actor.url + "'>" + post.actor.displayName + "</a> <a href='"+ post.object.url + "'> wrotes</a> via " + network + " at " + post.updated + ":<br><br>" + post.object.content - if not post.object.attachments? - _.each post.object.attachments, (attachment) -> - - if attachment.objectType == "photo" - parsed += "<br><img src='" + attachment.image.url + "'></img>" - - if attachment.objectType == "article" - parsed += "<br><a href='" + attachment.url + "'>"+ attachment.displayName + "</a>" - if not attachment.image? - parsed += "<br><img src='" + attachment.image.url + "'></img>" - - if attachment.objectType == "video" - parsed += "<br><a href='" + attachment.url + "'>"+ attachment.displayName + "</a>" - if not attachment.image? - parsed += "<br><img src='" + attachment.image.url + "'></img>" - - callback null, parsed - - if network is 'twitter' - - parsed = "<img src='" + post.user.profile_image_url + "'></img> <a href='https://twitter.com/account/redirect_by_id/" + post.user.id + "'>" + post.user.name + "</a> <a href='https://twitter.com/" + post.user.screen_name + "/status/" + post.id_str + "'> wrotes </a> via " + network + " at " + post.created_at + ":<br><br>" + post.text - _.each post.entities.media, (attachment) -> - parsed += "<br><img src='" + attachment.media_url + "'></img>" - callback null, parsed - - return - - sync = () -> # Do this every xx minutes @@ -124,7 +66,7 @@ interval = config.interval if not (interval?) interval = 15 * 60 * 1000 # 900 000 ms (15min) - + async.waterfall [ (callback) -> db.connect(config.params, callback) @@ -135,6 +77,3 @@ return sync() - -exports.postParser = postParser -exports.sync = sync
--- a/src/toESN.coffee Fri Jun 13 21:21:41 2014 +0200 +++ b/src/toESN.coffee Mon Jun 16 17:31:05 2014 +0200 @@ -22,6 +22,7 @@ "updated" ] indices: [ + "uid" "sourceUser" "sourcePost" "targetUser"
--- a/src/twitter.coffee Fri Jun 13 21:21:41 2014 +0200 +++ b/src/twitter.coffee Mon Jun 16 17:31:05 2014 +0200 @@ -28,6 +28,8 @@ FromESN = require("./fromESN") ToESN = require("./toESN") Sync = require("./sync") +util = require("util") +twitter = require("twitter") module.exports = (config) -> bridgeid = config.bridgeid @@ -65,6 +67,8 @@ separator = "?" authorization_endpoint + separator + "oauth_token=" + rt.token + return + getAccessToken: (site, rt, verifier, callback) -> oa = Twitter.getOAuth(site) oa.getOAuthAccessToken rt.token, rt.secret, verifier, callback @@ -89,13 +93,11 @@ return - sync: (user) -> + syncFromESN: (user) -> me = user.user_pumpio id = user.user_ESN.substr(0,user.user_ESN.indexOf('@')) token = user.oauth_token secret = user.extra_token - util = require("util") - twitter = require("twitter") twit = new twitter( consumer_key: client_id consumer_secret: client_secret @@ -109,13 +111,14 @@ Usermap.search {id: me + '_to_' + me}, callback (user, callback) -> if (user[0]?) + console.log "get feed from" + me Pump.getUserFeed(user[0],callback) else return (feed, callback) -> return if not feed? ti = new Date().getTime() - interval - async.eachSeries feed.items, ((post, callback) -> + async.eachLimit feed.items, 10, ((post, callback) -> # do for each post ts = Date.parse(post.updated) if (ts >= ti and post.verb is "post" or post.verb is "share") and (post.object.objectType is "note" or post.object.objectType is "image") and (Pump.isPublicActivity(post)) and (typeof post.object.deleted is "undefined") @@ -154,27 +157,43 @@ ), (err) -> callback null, 'done' ],(err, result) -> + console.log "done ToESN twitter user" + me + return + return + syncToESN: (user) -> + me = user.user_pumpio + id = user.user_ESN.substr(0,user.user_ESN.indexOf('@')) + token = user.oauth_token + secret = user.extra_token + twit = new twitter( + consumer_key: client_id + consumer_secret: client_secret + access_token_key: token + access_token_secret: secret + ) ###################################### # get tweets and post them to pump.io# ###################################### twit.verifyCredentials((data) -> #console.log util.inspect(data) + console.log "get twitter HomeTimeline " + me return ).getHomeTimeline {include_rts: true, count: 20 }, (data) -> - #console.log util.inspect(data) ti = (new Date().getTime() - interval) - async.each data, ((tweet,cb) -> + return if not data? or data.length is 0 + async.eachLimit data, 10, ((tweet, cb) -> async.waterfall [ (callback) -> - FromESN.search {uid: tweet.id + "@twitter_to_" + me, recipientUser: me}, callback + FromESN.search {uid: tweet.id + "@twitter_to_" + me}, callback (result, callback) -> return if result.length isnt 0 or tweet.user.id_str is id twitterdate = new Date(Date.parse(tweet.created_at.replace(/( +)/, " UTC$1"))) ts = Date.parse(twitterdate) if (ts >= ti) Sync.postParser tweet, null, 'twitter', callback - return + else + return (parsed, callback) -> Pump.postUser bridgeid, me, parsed, callback (pumppost, callback) -> @@ -189,12 +208,15 @@ , callback ], (err, result) -> cb() + return ), (err) -> + if (err) + console.log "Error: " + err + else + console.log "done ToESN twitter user" + me return return - parseTwitterDate : (text) -> - getOAuth: (site) -> new OAuth(request_token_endpoint, access_token_endpoint, client_id, client_secret, "1.0", site.url("/authorized-for-twitter"), "HMAC-SHA1", null, # nonce size; use default "User-Agent": site.userAgent()