commit: 24cafd73a2b644025e9aeaadf4fed46dd3ecea4d parent: 4a2fc2d444a80050ad9ba5e83aa5e69d3148ab95 Author: Eugen Rochko <eugen@zeonfederated.com> Date: Sat, 18 Nov 2017 00:16:48 +0100 Lists (#5703) * Add structure for lists * Add list timeline streaming API * Add list APIs, bind list-account relation to follow relation * Add API for adding/removing accounts from lists * Add pagination to lists API * Add pagination to list accounts API * Adjust scopes for new APIs - Creating and modifying lists merely requires "write" scope - Fetching information about lists merely requires "read" scope * Add test for wrong user context on list timeline * Clean up testsDiffstat:
68 files changed, 899 insertions(+), 268 deletions(-)
diff --git a/app/controllers/api/v1/lists/accounts_controller.rb b/app/controllers/api/v1/lists/accounts_controller.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +class Api::V1::Lists::AccountsController < Api::BaseController + before_action -> { doorkeeper_authorize! :read }, only: [:show] + before_action -> { doorkeeper_authorize! :write }, except: [:show] + + before_action :require_user! + before_action :set_list + + after_action :insert_pagination_headers, only: :show + + def show + @accounts = @list.accounts.paginate_by_max_id(limit_param(DEFAULT_ACCOUNTS_LIMIT), params[:max_id], params[:since_id]) + render json: @accounts, each_serializer: REST::AccountSerializer + end + + def create + ApplicationRecord.transaction do + list_accounts.each do |account| + @list.accounts << account + end + end + + render_empty + end + + def destroy + ListAccount.where(list: @list, account_id: account_ids).destroy_all + render_empty + end + + private + + def set_list + @list = List.where(account: current_account).find(params[:list_id]) + end + + def list_accounts + Account.find(account_ids) + end + + def account_ids + Array(resource_params[:account_ids]) + end + + def resource_params + params.permit(account_ids: []) + end + + def insert_pagination_headers + set_pagination_headers(next_path, prev_path) + end + + def next_path + if records_continue? + api_v1_list_accounts_url pagination_params(max_id: pagination_max_id) + end + end + + def prev_path + unless @accounts.empty? + api_v1_list_accounts_url pagination_params(since_id: pagination_since_id) + end + end + + def pagination_max_id + @accounts.last.id + end + + def pagination_since_id + @accounts.first.id + end + + def records_continue? + @accounts.size == limit_param(DEFAULT_ACCOUNTS_LIMIT) + end + + def pagination_params(core_params) + params.permit(:limit).merge(core_params) + end +end diff --git a/app/controllers/api/v1/lists_controller.rb b/app/controllers/api/v1/lists_controller.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +class Api::V1::ListsController < Api::BaseController + LISTS_LIMIT = 50 + + before_action -> { doorkeeper_authorize! :read }, only: [:index, :show] + before_action -> { doorkeeper_authorize! :write }, except: [:index, :show] + + before_action :require_user! + before_action :set_list, except: [:index, :create] + + after_action :insert_pagination_headers, only: :index + + def index + @lists = List.where(account: current_account).paginate_by_max_id(limit_param(LISTS_LIMIT), params[:max_id], params[:since_id]) + render json: @lists, each_serializer: REST::ListSerializer + end + + def show + render json: @list, serializer: REST::ListSerializer + end + + def create + @list = List.create!(list_params.merge(account: current_account)) + render json: @list, serializer: REST::ListSerializer + end + + def update + @list.update!(list_params) + render json: @list, serializer: REST::ListSerializer + end + + def destroy + @list.destroy! + render_empty + end + + private + + def set_list + @list = List.where(account: current_account).find(params[:id]) + end + + def list_params + params.permit(:title) + end + + def insert_pagination_headers + set_pagination_headers(next_path, prev_path) + end + + def next_path + if records_continue? + api_v1_lists_url pagination_params(max_id: pagination_max_id) + end + end + + def prev_path + unless @lists.empty? + api_v1_lists_url pagination_params(since_id: pagination_since_id) + end + end + + def pagination_max_id + @lists.last.id + end + + def pagination_since_id + @lists.first.id + end + + def records_continue? + @lists.size == limit_param(LISTS_LIMIT) + end + + def pagination_params(core_params) + params.permit(:limit).merge(core_params) + end +end diff --git a/app/controllers/api/v1/timelines/home_controller.rb b/app/controllers/api/v1/timelines/home_controller.rb @@ -31,7 +31,7 @@ class Api::V1::Timelines::HomeController < Api::BaseController end def account_home_feed - Feed.new(:home, current_account) + HomeFeed.new(current_account) end def insert_pagination_headers diff --git a/app/controllers/api/v1/timelines/list_controller.rb b/app/controllers/api/v1/timelines/list_controller.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +class Api::V1::Timelines::ListController < Api::BaseController + before_action -> { doorkeeper_authorize! :read } + before_action :require_user! + before_action :set_list + before_action :set_statuses + + after_action :insert_pagination_headers, unless: -> { @statuses.empty? } + + def show + render json: @statuses, + each_serializer: REST::StatusSerializer, + relationships: StatusRelationshipsPresenter.new(@statuses, current_user.account_id) + end + + private + + def set_list + @list = List.where(account: current_account).find(params[:id]) + end + + def set_statuses + @statuses = cached_list_statuses + end + + def cached_list_statuses + cache_collection list_statuses, Status + end + + def list_statuses + list_feed.get( + limit_param(DEFAULT_STATUSES_LIMIT), + params[:max_id], + params[:since_id] + ) + end + + def list_feed + ListFeed.new(@list) + end + + def insert_pagination_headers + set_pagination_headers(next_path, prev_path) + end + + def pagination_params(core_params) + params.permit(:limit).merge(core_params) + end + + def next_path + api_v1_timelines_list_url params[:id], pagination_params(max_id: pagination_max_id) + end + + def prev_path + api_v1_timelines_list_url params[:id], pagination_params(since_id: pagination_since_id) + end + + def pagination_max_id + @statuses.last.id + end + + def pagination_since_id + @statuses.first.id + end +end diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb @@ -26,34 +26,42 @@ class FeedManager end end - def push(timeline_type, account, status) - return false unless add_to_feed(timeline_type, account, status) - - trim(timeline_type, account.id) - - PushUpdateWorker.perform_async(account.id, status.id) if push_update_required?(timeline_type, account.id) - + def push_to_home(account, status) + return false unless add_to_feed(:home, account.id, status) + trim(:home, account.id) + PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}") if push_update_required?("timeline:#{account.id}") true end - def unpush(timeline_type, account, status) - return false unless remove_from_feed(timeline_type, account, status) + def unpush_from_home(account, status) + return false unless remove_from_feed(:home, account.id, status) + Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) + true + end - payload = Oj.dump(event: :delete, payload: status.id.to_s) - Redis.current.publish("timeline:#{account.id}", payload) + def push_to_list(list, status) + return false unless add_to_feed(:list, list.id, status) + trim(:list, list.id) + PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}") + true + end + def unpush_from_list(list, status) + return false unless remove_from_feed(:list, list.id, status) + Redis.current.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) true end def trim(type, account_id) timeline_key = key(type, account_id) - reblog_key = key(type, account_id, 'reblogs') + reblog_key = key(type, account_id, 'reblogs') + # Remove any items past the MAX_ITEMS'th entry in our feed redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s) # Get the score of the REBLOG_FALLOFF'th item in our feed, and stop # tracking anything after it for deduplication purposes. - falloff_rank = FeedManager::REBLOG_FALLOFF - 1 + falloff_rank = FeedManager::REBLOG_FALLOFF - 1 falloff_range = redis.zrevrange(timeline_key, falloff_rank, falloff_rank, with_scores: true) falloff_score = falloff_range&.first&.last&.to_i || 0 @@ -69,10 +77,6 @@ class FeedManager end end - def push_update_required?(timeline_type, account_id) - timeline_type != :home || redis.get("subscribed:timeline:#{account_id}").present? - end - def merge_into_timeline(from_account, into_account) timeline_key = key(:home, into_account.id) query = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4) @@ -84,28 +88,28 @@ class FeedManager query.each do |status| next if status.direct_visibility? || filter?(:home, status, into_account) - add_to_feed(:home, into_account, status) + add_to_feed(:home, into_account.id, status) end trim(:home, into_account.id) end def unmerge_from_timeline(from_account, into_account) - timeline_key = key(:home, into_account.id) + timeline_key = key(:home, into_account.id) oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0 from_account.statuses.select('id, reblog_of_id').where('id > ?', oldest_home_score).reorder(nil).find_each do |status| - remove_from_feed(:home, into_account, status) + remove_from_feed(:home, into_account.id, status) end end def clear_from_timeline(account, target_account) - timeline_key = key(:home, account.id) + timeline_key = key(:home, account.id) timeline_status_ids = redis.zrange(timeline_key, 0, -1) - target_statuses = Status.where(id: timeline_status_ids, account: target_account) + target_statuses = Status.where(id: timeline_status_ids, account: target_account) target_statuses.each do |status| - unpush(:home, account, status) + unpush_from_home(account, status) end end @@ -122,7 +126,7 @@ class FeedManager statuses.each do |status| next if filter_from_home?(status, account) - added += 1 if add_to_feed(:home, account, status) + added += 1 if add_to_feed(:home, account.id, status) end break unless added.zero? @@ -137,6 +141,10 @@ class FeedManager Redis.current end + def push_update_required?(timeline_id) + redis.exists("subscribed:#{timeline_id}") + end + def filter_from_home?(status, receiver_id) return false if receiver_id == status.account_id return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) @@ -182,9 +190,9 @@ class FeedManager # added, and false if it was not added to the feed. Note that this is # an internal helper: callers must call trim or push updates if # either action is appropriate. - def add_to_feed(timeline_type, account, status) - timeline_key = key(timeline_type, account.id) - reblog_key = key(timeline_type, account.id, 'reblogs') + def add_to_feed(timeline_type, account_id, status) + timeline_key = key(timeline_type, account_id) + reblog_key = key(timeline_type, account_id, 'reblogs') if status.reblog? # If the original status or a reblog of it is within @@ -195,6 +203,7 @@ class FeedManager return false if !rank.nil? && rank < FeedManager::REBLOG_FALLOFF reblog_rank = redis.zrevrank(reblog_key, status.reblog_of_id) + if reblog_rank.nil? # This is not something we've already seen reblogged, so we # can just add it to the feed (and note that we're @@ -205,7 +214,7 @@ class FeedManager # Another reblog of the same status was already in the # REBLOG_FALLOFF most recent statuses, so we note that this # is an "extra" reblog, by storing it in reblog_set_key. - reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}") + reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}") redis.sadd(reblog_set_key, status.id) return false end @@ -220,8 +229,8 @@ class FeedManager # with reblogs, and returning true if a status was removed. As with # `add_to_feed`, this does not trigger push updates, so callers must # do so if appropriate. - def remove_from_feed(timeline_type, account, status) - timeline_key = key(timeline_type, account.id) + def remove_from_feed(timeline_type, account_id, status) + timeline_key = key(timeline_type, account_id) if status.reblog? # 1. If the reblogging status is not in the feed, stop. @@ -229,7 +238,7 @@ class FeedManager return false if status_rank.nil? # 2. Remove reblog from set of this status's reblogs. - reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}") + reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}") redis.srem(reblog_set_key, status.id) # 3. Re-insert another reblog or original into the feed if one @@ -244,7 +253,7 @@ class FeedManager # (outside conditional) else # If the original is getting deleted, no use for reblog references - redis.del(key(timeline_type, account.id, "reblogs:#{status.id}")) + redis.del(key(timeline_type, account_id, "reblogs:#{status.id}")) end redis.zrem(timeline_key, status.id) diff --git a/app/models/account.rb b/app/models/account.rb @@ -3,7 +3,7 @@ # # Table name: accounts # -# id :bigint not null, primary key +# id :integer not null, primary key # username :string default(""), not null # domain :string # secret :string default(""), not null @@ -53,6 +53,7 @@ class Account < ApplicationRecord include AccountInteractions include Attachmentable include Remotable + include Paginable enum protocol: [:ostatus, :activitypub] @@ -95,6 +96,10 @@ class Account < ApplicationRecord has_many :account_moderation_notes, dependent: :destroy has_many :targeted_moderation_notes, class_name: 'AccountModerationNote', foreign_key: :target_account_id, dependent: :destroy + # Lists + has_many :list_accounts, inverse_of: :account, dependent: :destroy + has_many :lists, through: :list_accounts + scope :remote, -> { where.not(domain: nil) } scope :local, -> { where(domain: nil) } scope :without_followers, -> { where(followers_count: 0) } diff --git a/app/models/account_domain_block.rb b/app/models/account_domain_block.rb @@ -3,11 +3,11 @@ # # Table name: account_domain_blocks # +# id :integer not null, primary key # domain :string # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint -# id :bigint not null, primary key +# account_id :integer # class AccountDomainBlock < ApplicationRecord diff --git a/app/models/account_moderation_note.rb b/app/models/account_moderation_note.rb @@ -3,10 +3,10 @@ # # Table name: account_moderation_notes # -# id :bigint not null, primary key +# id :integer not null, primary key # content :text not null -# account_id :bigint not null -# target_account_id :bigint not null +# account_id :integer not null +# target_account_id :integer not null # created_at :datetime not null # updated_at :datetime not null # diff --git a/app/models/block.rb b/app/models/block.rb @@ -3,11 +3,11 @@ # # Table name: blocks # +# id :integer not null, primary key # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint not null -# id :bigint not null, primary key -# target_account_id :bigint not null +# account_id :integer not null +# target_account_id :integer not null # class Block < ApplicationRecord diff --git a/app/models/conversation.rb b/app/models/conversation.rb @@ -3,7 +3,7 @@ # # Table name: conversations # -# id :bigint not null, primary key +# id :integer not null, primary key # uri :string # created_at :datetime not null # updated_at :datetime not null diff --git a/app/models/conversation_mute.rb b/app/models/conversation_mute.rb @@ -3,9 +3,9 @@ # # Table name: conversation_mutes # -# conversation_id :bigint not null -# account_id :bigint not null -# id :bigint not null, primary key +# id :integer not null, primary key +# conversation_id :integer not null +# account_id :integer not null # class ConversationMute < ApplicationRecord diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb @@ -3,7 +3,7 @@ # # Table name: custom_emojis # -# id :bigint not null, primary key +# id :integer not null, primary key # shortcode :string default(""), not null # domain :string # image_file_name :string diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb @@ -3,12 +3,12 @@ # # Table name: domain_blocks # +# id :integer not null, primary key # domain :string default(""), not null # created_at :datetime not null # updated_at :datetime not null # severity :integer default("silence") # reject_media :boolean default(FALSE), not null -# id :bigint not null, primary key # class DomainBlock < ApplicationRecord diff --git a/app/models/email_domain_block.rb b/app/models/email_domain_block.rb @@ -3,7 +3,7 @@ # # Table name: email_domain_blocks # -# id :bigint not null, primary key +# id :integer not null, primary key # domain :string default(""), not null # created_at :datetime not null # updated_at :datetime not null diff --git a/app/models/favourite.rb b/app/models/favourite.rb @@ -3,11 +3,11 @@ # # Table name: favourites # +# id :integer not null, primary key # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint not null -# id :bigint not null, primary key -# status_id :bigint not null +# account_id :integer not null +# status_id :integer not null # class Favourite < ApplicationRecord diff --git a/app/models/feed.rb b/app/models/feed.rb @@ -1,36 +1,27 @@ # frozen_string_literal: true class Feed - def initialize(type, account) - @type = type - @account = account + def initialize(type, id) + @type = type + @id = id end def get(limit, max_id = nil, since_id = nil) - if redis.exists("account:#{@account.id}:regeneration") - from_database(limit, max_id, since_id) - else - from_redis(limit, max_id, since_id) - end + from_redis(limit, max_id, since_id) end - private + protected def from_redis(limit, max_id, since_id) max_id = '+inf' if max_id.blank? since_id = '-inf' if since_id.blank? unhydrated = redis.zrevrangebyscore(key, "(#{max_id}", "(#{since_id}", limit: [0, limit], with_scores: true).map(&:first).map(&:to_i) - Status.where(id: unhydrated).cache_ids - end - def from_database(limit, max_id, since_id) - Status.as_home_timeline(@account) - .paginate_by_max_id(limit, max_id, since_id) - .reject { |status| FeedManager.instance.filter?(:home, status, @account.id) } + Status.where(id: unhydrated).cache_ids end def key - FeedManager.instance.key(@type, @account.id) + FeedManager.instance.key(@type, @id) end def redis diff --git a/app/models/follow.rb b/app/models/follow.rb @@ -3,11 +3,11 @@ # # Table name: follows # +# id :integer not null, primary key # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint not null -# id :bigint not null, primary key -# target_account_id :bigint not null +# account_id :integer not null +# target_account_id :integer not null # class Follow < ApplicationRecord diff --git a/app/models/follow_request.rb b/app/models/follow_request.rb @@ -3,11 +3,11 @@ # # Table name: follow_requests # +# id :integer not null, primary key # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint not null -# id :bigint not null, primary key -# target_account_id :bigint not null +# account_id :integer not null +# target_account_id :integer not null # class FollowRequest < ApplicationRecord diff --git a/app/models/home_feed.rb b/app/models/home_feed.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class HomeFeed < Feed + def initialize(account) + @type = :home + @id = account.id + @account = account + end + + def get(limit, max_id = nil, since_id = nil) + if redis.exists("account:#{@account.id}:regeneration") + from_database(limit, max_id, since_id) + else + super + end + end + + private + + def from_database(limit, max_id, since_id) + Status.as_home_timeline(@account) + .paginate_by_max_id(limit, max_id, since_id) + .reject { |status| FeedManager.instance.filter?(:home, status, @account.id) } + end +end diff --git a/app/models/import.rb b/app/models/import.rb @@ -3,6 +3,7 @@ # # Table name: imports # +# id :integer not null, primary key # type :integer not null # approved :boolean default(FALSE), not null # created_at :datetime not null @@ -11,8 +12,7 @@ # data_content_type :string # data_file_size :integer # data_updated_at :datetime -# account_id :bigint not null -# id :bigint not null, primary key +# account_id :integer not null # class Import < ApplicationRecord diff --git a/app/models/list.rb b/app/models/list.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: lists +# +# id :integer not null, primary key +# account_id :integer +# title :string default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class List < ApplicationRecord + include Paginable + + belongs_to :account + + has_many :list_accounts, inverse_of: :list, dependent: :destroy + has_many :accounts, through: :list_accounts + + validates :title, presence: true +end diff --git a/app/models/list_account.rb b/app/models/list_account.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: list_accounts +# +# id :integer not null, primary key +# list_id :integer not null +# account_id :integer not null +# follow_id :integer not null +# + +class ListAccount < ApplicationRecord + belongs_to :list, required: true + belongs_to :account, required: true + belongs_to :follow, required: true + + before_validation :set_follow + + private + + def set_follow + self.follow = Follow.find_by(account_id: list.account_id, target_account_id: account.id) + end +end diff --git a/app/models/list_feed.rb b/app/models/list_feed.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +class ListFeed < Feed + def initialize(list) + @type = :list + @id = list.id + end +end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb @@ -3,19 +3,19 @@ # # Table name: media_attachments # -# id :bigint not null, primary key -# status_id :bigint +# id :integer not null, primary key +# status_id :integer # file_file_name :string # file_content_type :string # file_file_size :integer # file_updated_at :datetime # remote_url :string default(""), not null -# account_id :bigint # created_at :datetime not null # updated_at :datetime not null # shortcode :string # type :integer default("image"), not null # file_meta :json +# account_id :integer # description :text # diff --git a/app/models/mention.rb b/app/models/mention.rb @@ -3,11 +3,11 @@ # # Table name: mentions # -# status_id :bigint +# id :integer not null, primary key +# status_id :integer # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint -# id :bigint not null, primary key +# account_id :integer # class Mention < ApplicationRecord diff --git a/app/models/notification.rb b/app/models/notification.rb @@ -3,13 +3,13 @@ # # Table name: notifications # -# id :bigint not null, primary key -# account_id :bigint -# activity_id :bigint +# id :integer not null, primary key +# activity_id :integer # activity_type :string # created_at :datetime not null # updated_at :datetime not null -# from_account_id :bigint +# account_id :integer +# from_account_id :integer # class Notification < ApplicationRecord diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb @@ -3,7 +3,7 @@ # # Table name: preview_cards # -# id :bigint not null, primary key +# id :integer not null, primary key # url :string default(""), not null # title :string default(""), not null # description :string default(""), not null diff --git a/app/models/report.rb b/app/models/report.rb @@ -3,15 +3,15 @@ # # Table name: reports # +# id :integer not null, primary key # status_ids :integer default([]), not null, is an Array # comment :text default(""), not null # action_taken :boolean default(FALSE), not null # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint not null -# action_taken_by_account_id :bigint -# id :bigint not null, primary key -# target_account_id :bigint not null +# account_id :integer not null +# action_taken_by_account_id :integer +# target_account_id :integer not null # class Report < ApplicationRecord diff --git a/app/models/session_activation.rb b/app/models/session_activation.rb @@ -3,15 +3,15 @@ # # Table name: session_activations # -# id :bigint not null, primary key -# user_id :bigint not null +# id :integer not null, primary key # session_id :string not null # created_at :datetime not null # updated_at :datetime not null # user_agent :string default(""), not null # ip :inet -# access_token_id :bigint -# web_push_subscription_id :bigint +# access_token_id :integer +# user_id :integer not null +# web_push_subscription_id :integer # # id :bigint not null, primary key diff --git a/app/models/setting.rb b/app/models/setting.rb @@ -3,13 +3,13 @@ # # Table name: settings # +# id :integer not null, primary key # var :string not null # value :text # thing_type :string # created_at :datetime # updated_at :datetime -# id :bigint not null, primary key -# thing_id :bigint +# thing_id :integer # class Setting < RailsSettings::Base diff --git a/app/models/site_upload.rb b/app/models/site_upload.rb @@ -3,7 +3,7 @@ # # Table name: site_uploads # -# id :bigint not null, primary key +# id :integer not null, primary key # var :string default(""), not null # file_file_name :string # file_content_type :string diff --git a/app/models/status.rb b/app/models/status.rb @@ -3,26 +3,26 @@ # # Table name: statuses # -# id :bigint not null, primary key +# id :integer not null, primary key # uri :string -# account_id :bigint not null # text :text default(""), not null # created_at :datetime not null # updated_at :datetime not null -# in_reply_to_id :bigint -# reblog_of_id :bigint +# in_reply_to_id :integer +# reblog_of_id :integer # url :string # sensitive :boolean default(FALSE), not null # visibility :integer default("public"), not null -# in_reply_to_account_id :bigint -# application_id :bigint # spoiler_text :text default(""), not null # reply :boolean default(FALSE), not null # favourites_count :integer default(0), not null # reblogs_count :integer default(0), not null # language :string -# conversation_id :bigint +# conversation_id :integer # local :boolean +# account_id :integer not null +# application_id :integer +# in_reply_to_account_id :integer # class Status < ApplicationRecord diff --git a/app/models/status_pin.rb b/app/models/status_pin.rb @@ -3,9 +3,9 @@ # # Table name: status_pins # -# id :bigint not null, primary key -# account_id :bigint not null -# status_id :bigint not null +# id :integer not null, primary key +# account_id :integer not null +# status_id :integer not null # created_at :datetime not null # updated_at :datetime not null # diff --git a/app/models/stream_entry.rb b/app/models/stream_entry.rb @@ -3,13 +3,13 @@ # # Table name: stream_entries # -# activity_id :bigint +# id :integer not null, primary key +# activity_id :integer # activity_type :string # created_at :datetime not null # updated_at :datetime not null # hidden :boolean default(FALSE), not null -# account_id :bigint -# id :bigint not null, primary key +# account_id :integer # class StreamEntry < ApplicationRecord diff --git a/app/models/subscription.rb b/app/models/subscription.rb @@ -3,6 +3,7 @@ # # Table name: subscriptions # +# id :integer not null, primary key # callback_url :string default(""), not null # secret :string # expires_at :datetime @@ -11,8 +12,7 @@ # updated_at :datetime not null # last_successful_delivery_at :datetime # domain :string -# account_id :bigint not null -# id :bigint not null, primary key +# account_id :integer not null # class Subscription < ApplicationRecord diff --git a/app/models/tag.rb b/app/models/tag.rb @@ -3,7 +3,7 @@ # # Table name: tags # -# id :bigint not null, primary key +# id :integer not null, primary key # name :string default(""), not null # created_at :datetime not null # updated_at :datetime not null diff --git a/app/models/user.rb b/app/models/user.rb @@ -3,7 +3,7 @@ # # Table name: users # -# id :bigint not null, primary key +# id :integer not null, primary key # email :string default(""), not null # created_at :datetime not null # updated_at :datetime not null @@ -30,7 +30,7 @@ # last_emailed_at :datetime # otp_backup_codes :string is an Array # filtered_languages :string default([]), not null, is an Array -# account_id :bigint not null +# account_id :integer not null # disabled :boolean default(FALSE), not null # moderator :boolean default(FALSE), not null # diff --git a/app/models/web/push_subscription.rb b/app/models/web/push_subscription.rb @@ -3,7 +3,7 @@ # # Table name: web_push_subscriptions # -# id :bigint not null, primary key +# id :integer not null, primary key # endpoint :string not null # key_p256dh :string not null # key_auth :string not null diff --git a/app/models/web/setting.rb b/app/models/web/setting.rb @@ -3,11 +3,11 @@ # # Table name: web_settings # +# id :integer not null, primary key # data :json # created_at :datetime not null # updated_at :datetime not null -# id :bigint not null, primary key -# user_id :bigint +# user_id :integer # class Web::Setting < ApplicationRecord diff --git a/app/serializers/rest/list_serializer.rb b/app/serializers/rest/list_serializer.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class REST::ListSerializer < ActiveModel::Serializer + attributes :id, :title +end diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb @@ -30,6 +30,7 @@ class BatchedRemoveStatusService < BaseService account = account_statuses.first.account unpush_from_home_timelines(account, account_statuses) + unpush_from_list_timelines(account, account_statuses) if account.local? batch_stream_entries(account, account_statuses) @@ -79,7 +80,15 @@ class BatchedRemoveStatusService < BaseService recipients.each do |follower| statuses.each do |status| - FeedManager.instance.unpush(:home, follower, status) + FeedManager.instance.unpush_from_home(follower, status) + end + end + end + + def unpush_from_list_timelines(account, statuses) + account.lists.select(:id, :account_id).each do |list| + statuses.each do |status| + FeedManager.instance.unpush_from_list(list, status) end end end diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb @@ -14,6 +14,7 @@ class FanOutOnWriteService < BaseService deliver_to_mentioned_followers(status) else deliver_to_followers(status) + deliver_to_lists(status) end return if status.account.silenced? || !status.public_visibility? || status.reblog? @@ -30,7 +31,7 @@ class FanOutOnWriteService < BaseService def deliver_to_self(status) Rails.logger.debug "Delivering status #{status.id} to author" - FeedManager.instance.push(:home, status.account, status) + FeedManager.instance.push_to_home(status.account, status) end def deliver_to_followers(status) @@ -38,7 +39,17 @@ class FanOutOnWriteService < BaseService status.account.followers.where(domain: nil).joins(:user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_in_batches do |followers| FeedInsertWorker.push_bulk(followers) do |follower| - [status.id, follower.id] + [status.id, follower.id, :home] + end + end + end + + def deliver_to_lists(status) + Rails.logger.debug "Delivering status #{status.id} to lists" + + status.account.lists.joins(account: :user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_in_batches do |lists| + FeedInsertWorker.push_bulk(lists) do |list| + [status.id, list.id, :list] end end end @@ -49,7 +60,7 @@ class FanOutOnWriteService < BaseService status.mentions.includes(:account).each do |mention| mentioned_account = mention.account next if !mentioned_account.local? || !mentioned_account.following?(status.account) || FeedManager.instance.filter?(:home, status, mention.account_id) - FeedManager.instance.push(:home, mentioned_account, status) + FeedManager.instance.push_to_home(mentioned_account, status) end end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb @@ -14,6 +14,7 @@ class RemoveStatusService < BaseService remove_from_self if status.account.local? remove_from_followers + remove_from_lists remove_from_affected remove_reblogs remove_from_hashtags @@ -30,12 +31,18 @@ class RemoveStatusService < BaseService private def remove_from_self - unpush(:home, @account, @status) + FeedManager.instance.unpush_from_home(@account, @status) end def remove_from_followers @account.followers.local.find_each do |follower| - unpush(:home, follower, @status) + FeedManager.instance.unpush_from_home(follower, @status) + end + end + + def remove_from_lists + @account.lists.select(:id, :account_id).find_each do |list| + FeedManager.instance.unpush_from_list(list, @status) end end @@ -101,10 +108,6 @@ class RemoveStatusService < BaseService end end - def unpush(type, receiver, status) - FeedManager.instance.unpush(type, receiver, status) - end - def remove_from_hashtags return unless @status.public_visibility? diff --git a/app/workers/feed_insert_worker.rb b/app/workers/feed_insert_worker.rb @@ -3,34 +3,41 @@ class FeedInsertWorker include Sidekiq::Worker - attr_reader :status, :follower - - def perform(status_id, follower_id) - @status = Status.find_by(id: status_id) - @follower = Account.find_by(id: follower_id) + def perform(status_id, id, type = :home) + @type = type.to_sym + @status = Status.find(status_id) + + case @type + when :home + @follower = Account.find(id) + when :list + @list = List.find(id) + @follower = @list.account + end check_and_insert + rescue ActiveRecord::RecordNotFound + true end private def check_and_insert - if records_available? - perform_push unless feed_filtered? - else - true - end - end - - def records_available? - status.present? && follower.present? + perform_push unless feed_filtered? end def feed_filtered? - FeedManager.instance.filter?(:home, status, follower.id) + # Note: Lists are a variation of home, so the filtering rules + # of home apply to both + FeedManager.instance.filter?(:home, @status, @follower.id) end def perform_push - FeedManager.instance.push(:home, follower, status) + case @type + when :home + FeedManager.instance.push_to_home(@follower, @status) + when :list + FeedManager.instance.push_to_list(@list, @status) + end end end diff --git a/app/workers/push_update_worker.rb b/app/workers/push_update_worker.rb @@ -3,12 +3,13 @@ class PushUpdateWorker include Sidekiq::Worker - def perform(account_id, status_id) - account = Account.find(account_id) - status = Status.find(status_id) - message = InlineRenderer.render(status, account, :status) + def perform(account_id, status_id, timeline_id = nil) + account = Account.find(account_id) + status = Status.find(status_id) + message = InlineRenderer.render(status, account, :status) + timeline_id = "timeline:#{account.id}" if timeline_id.nil? - Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i)) + Redis.current.publish(timeline_id, Oj.dump(event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i)) rescue ActiveRecord::RecordNotFound true end diff --git a/config/routes.rb b/config/routes.rb @@ -212,6 +212,7 @@ Rails.application.routes.draw do resource :home, only: :show, controller: :home resource :public, only: :show, controller: :public resources :tag, only: :show + resources :list, only: :show end resources :streaming, only: [:index] @@ -270,6 +271,10 @@ Rails.application.routes.draw do post :unmute end end + + resources :lists, only: [:index, :create, :show, :update, :destroy] do + resource :accounts, only: [:show, :create, :destroy], controller: 'lists/accounts' + end end namespace :web do diff --git a/db/migrate/20171114231651_create_lists.rb b/db/migrate/20171114231651_create_lists.rb @@ -0,0 +1,10 @@ +class CreateLists < ActiveRecord::Migration[5.1] + def change + create_table :lists do |t| + t.references :account, foreign_key: { on_delete: :cascade } + t.string :title, null: false, default: '' + + t.timestamps + end + end +end diff --git a/db/migrate/20171116161857_create_list_accounts.rb b/db/migrate/20171116161857_create_list_accounts.rb @@ -0,0 +1,12 @@ +class CreateListAccounts < ActiveRecord::Migration[5.1] + def change + create_table :list_accounts do |t| + t.belongs_to :list, foreign_key: { on_delete: :cascade }, null: false + t.belongs_to :account, foreign_key: { on_delete: :cascade }, null: false + t.belongs_to :follow, foreign_key: { on_delete: :cascade }, null: false + end + + add_index :list_accounts, [:account_id, :list_id], unique: true + add_index :list_accounts, [:list_id, :account_id] + end +end diff --git a/db/schema.rb b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20171114080328) do +ActiveRecord::Schema.define(version: 20171116161857) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -170,6 +170,25 @@ ActiveRecord::Schema.define(version: 20171114080328) do t.bigint "account_id", null: false end + create_table "list_accounts", force: :cascade do |t| + t.bigint "list_id", null: false + t.bigint "account_id", null: false + t.bigint "follow_id", null: false + t.index ["account_id", "list_id"], name: "index_list_accounts_on_account_id_and_list_id", unique: true + t.index ["account_id"], name: "index_list_accounts_on_account_id" + t.index ["follow_id"], name: "index_list_accounts_on_follow_id" + t.index ["list_id", "account_id"], name: "index_list_accounts_on_list_id_and_account_id" + t.index ["list_id"], name: "index_list_accounts_on_list_id" + end + + create_table "lists", force: :cascade do |t| + t.bigint "account_id" + t.string "title", default: "", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_lists_on_account_id" + end + create_table "media_attachments", force: :cascade do |t| t.bigint "status_id" t.string "file_file_name" @@ -478,6 +497,10 @@ ActiveRecord::Schema.define(version: 20171114080328) do add_foreign_key "follows", "accounts", column: "target_account_id", name: "fk_745ca29eac", on_delete: :cascade add_foreign_key "follows", "accounts", name: "fk_32ed1b5560", on_delete: :cascade add_foreign_key "imports", "accounts", name: "fk_6db1b6e408", on_delete: :cascade + add_foreign_key "list_accounts", "accounts", on_delete: :cascade + add_foreign_key "list_accounts", "follows", on_delete: :cascade + add_foreign_key "list_accounts", "lists", on_delete: :cascade + add_foreign_key "lists", "accounts", on_delete: :cascade add_foreign_key "media_attachments", "accounts", name: "fk_96dd81e81b", on_delete: :nullify add_foreign_key "media_attachments", "statuses", on_delete: :nullify add_foreign_key "mentions", "accounts", name: "fk_970d43f9d1", on_delete: :cascade diff --git a/spec/controllers/api/v1/lists/accounts_controller_spec.rb b/spec/controllers/api/v1/lists/accounts_controller_spec.rb @@ -0,0 +1,54 @@ +require 'rails_helper' + +describe Api::V1::Lists::AccountsController do + render_views + + let(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read write') } + let(:list) { Fabricate(:list, account: user.account) } + + before do + follow = Fabricate(:follow, account: user.account) + list.accounts << follow.target_account + allow(controller).to receive(:doorkeeper_token) { token } + end + + describe 'GET #index' do + it 'returns http success' do + get :show, params: { list_id: list.id } + + expect(response).to have_http_status(:success) + end + end + + describe 'POST #create' do + let(:bob) { Fabricate(:account, username: 'bob') } + + before do + user.account.follow!(bob) + post :create, params: { list_id: list.id, account_ids: [bob.id] } + end + + it 'returns http success' do + expect(response).to have_http_status(:success) + end + + it 'adds account to the list' do + expect(list.accounts.include?(bob)).to be true + end + end + + describe 'DELETE #destroy' do + before do + delete :destroy, params: { list_id: list.id, account_ids: [list.accounts.first.id] } + end + + it 'returns http success' do + expect(response).to have_http_status(:success) + end + + it 'removes account from the list' do + expect(list.accounts.count).to eq 0 + end + end +end diff --git a/spec/controllers/api/v1/lists_controller_spec.rb b/spec/controllers/api/v1/lists_controller_spec.rb @@ -0,0 +1,68 @@ +require 'rails_helper' + +RSpec.describe Api::V1::ListsController, type: :controller do + render_views + + let!(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) } + let!(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read write') } + let!(:list) { Fabricate(:list, account: user.account) } + + before { allow(controller).to receive(:doorkeeper_token) { token } } + + describe 'GET #index' do + it 'returns http success' do + get :index + expect(response).to have_http_status(:success) + end + end + + describe 'GET #show' do + it 'returns http success' do + get :show, params: { id: list.id } + expect(response).to have_http_status(:success) + end + end + + describe 'POST #create' do + before do + post :create, params: { title: 'Foo bar' } + end + + it 'returns http success' do + expect(response).to have_http_status(:success) + end + + it 'creates list' do + expect(List.where(account: user.account).count).to eq 2 + expect(List.last.title).to eq 'Foo bar' + end + end + + describe 'PUT #update' do + before do + put :update, params: { id: list.id, title: 'Updated title' } + end + + it 'returns http success' do + expect(response).to have_http_status(:success) + end + + it 'updates the list' do + expect(list.reload.title).to eq 'Updated title' + end + end + + describe 'DELETE #destroy' do + before do + delete :destroy, params: { id: list.id } + end + + it 'returns http success' do + expect(response).to have_http_status(:success) + end + + it 'deletes the list' do + expect(List.find_by(id: list.id)).to be_nil + end + end +end diff --git a/spec/controllers/api/v1/timelines/list_controller_spec.rb b/spec/controllers/api/v1/timelines/list_controller_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Api::V1::Timelines::ListController do + render_views + + let(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) } + let(:list) { Fabricate(:list, account: user.account) } + + before do + allow(controller).to receive(:doorkeeper_token) { token } + end + + context 'with a user context' do + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read') } + + describe 'GET #show' do + before do + follow = Fabricate(:follow, account: user.account) + list.accounts << follow.target_account + PostStatusService.new.call(follow.target_account, 'New status for user home timeline.') + end + + it 'returns http success' do + get :show, params: { id: list.id } + expect(response).to have_http_status(:success) + end + end + end + + context 'with the wrong user context' do + let(:other_user) { Fabricate(:user, account: Fabricate(:account, username: 'bob')) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: other_user.id, scopes: 'read') } + + describe 'GET #show' do + it 'returns http not found' do + get :show, params: { id: list.id } + expect(response).to have_http_status(:not_found) + end + end + end + + context 'without a user context' do + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: 'read') } + + describe 'GET #show' do + it 'returns http unprocessable entity' do + get :show, params: { id: list.id } + + expect(response).to have_http_status(:unprocessable_entity) + expect(response.headers['Link']).to be_nil + end + end + end +end diff --git a/spec/controllers/api/v1/timelines/tag_controller_spec.rb b/spec/controllers/api/v1/timelines/tag_controller_spec.rb @@ -5,7 +5,7 @@ require 'rails_helper' describe Api::V1::Timelines::TagController do render_views - let(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) } + let(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) } before do allow(controller).to receive(:doorkeeper_token) { token } diff --git a/spec/fabricators/list_account_fabricator.rb b/spec/fabricators/list_account_fabricator.rb @@ -0,0 +1,5 @@ +Fabricator(:list_account) do + list nil + account nil + follow nil +end diff --git a/spec/fabricators/list_fabricator.rb b/spec/fabricators/list_fabricator.rb @@ -0,0 +1,4 @@ +Fabricator(:list) do + account nil + title "MyString" +end diff --git a/spec/lib/feed_manager_spec.rb b/spec/lib/feed_manager_spec.rb @@ -148,21 +148,11 @@ RSpec.describe FeedManager do account = Fabricate(:account) status = Fabricate(:status) members = FeedManager::MAX_ITEMS.times.map { |count| [count, count] } - Redis.current.zadd("feed:type:#{account.id}", members) + Redis.current.zadd("feed:home:#{account.id}", members) - FeedManager.instance.push('type', account, status) + FeedManager.instance.push_to_home(account, status) - expect(Redis.current.zcard("feed:type:#{account.id}")).to eq FeedManager::MAX_ITEMS - end - - it 'sends push updates for non-home timelines' do - account = Fabricate(:account) - status = Fabricate(:status) - allow(Redis.current).to receive_messages(publish: nil) - - FeedManager.instance.push('type', account, status) - - expect(Redis.current).to have_received(:publish).with("timeline:#{account.id}", any_args).at_least(:once) + expect(Redis.current.zcard("feed:home:#{account.id}")).to eq FeedManager::MAX_ITEMS end context 'reblogs' do @@ -171,7 +161,7 @@ RSpec.describe FeedManager do reblogged = Fabricate(:status) reblog = Fabricate(:status, reblog: reblogged) - expect(FeedManager.instance.push('type', account, reblog)).to be true + expect(FeedManager.instance.push_to_home(account, reblog)).to be true end it 'does not save a new reblog of a recent status' do @@ -179,9 +169,9 @@ RSpec.describe FeedManager do reblogged = Fabricate(:status) reblog = Fabricate(:status, reblog: reblogged) - FeedManager.instance.push('type', account, reblogged) + FeedManager.instance.push_to_home(account, reblogged) - expect(FeedManager.instance.push('type', account, reblog)).to be false + expect(FeedManager.instance.push_to_home(account, reblog)).to be false end it 'saves a new reblog of an old status' do @@ -189,14 +179,14 @@ RSpec.describe FeedManager do reblogged = Fabricate(:status) reblog = Fabricate(:status, reblog: reblogged) - FeedManager.instance.push('type', account, reblogged) + FeedManager.instance.push_to_home(account, reblogged) # Fill the feed with intervening statuses FeedManager::REBLOG_FALLOFF.times do - FeedManager.instance.push('type', account, Fabricate(:status)) + FeedManager.instance.push_to_home(account, Fabricate(:status)) end - expect(FeedManager.instance.push('type', account, reblog)).to be true + expect(FeedManager.instance.push_to_home(account, reblog)).to be true end it 'does not save a new reblog of a recently-reblogged status' do @@ -205,10 +195,10 @@ RSpec.describe FeedManager do reblogs = 2.times.map { Fabricate(:status, reblog: reblogged) } # The first reblog will be accepted - FeedManager.instance.push('type', account, reblogs.first) + FeedManager.instance.push_to_home(account, reblogs.first) # The second reblog should be ignored - expect(FeedManager.instance.push('type', account, reblogs.last)).to be false + expect(FeedManager.instance.push_to_home(account, reblogs.last)).to be false end it 'does not save a new reblog of a multiply-reblogged-then-unreblogged status' do @@ -217,14 +207,14 @@ RSpec.describe FeedManager do reblogs = 3.times.map { Fabricate(:status, reblog: reblogged) } # Accept the reblogs - FeedManager.instance.push('type', account, reblogs[0]) - FeedManager.instance.push('type', account, reblogs[1]) + FeedManager.instance.push_to_home(account, reblogs[0]) + FeedManager.instance.push_to_home(account, reblogs[1]) # Unreblog the first one - FeedManager.instance.unpush('type', account, reblogs[0]) + FeedManager.instance.unpush_from_home(account, reblogs[0]) # The last reblog should still be ignored - expect(FeedManager.instance.push('type', account, reblogs.last)).to be false + expect(FeedManager.instance.push_to_home(account, reblogs.last)).to be false end it 'saves a new reblog of a long-ago-reblogged status' do @@ -233,15 +223,15 @@ RSpec.describe FeedManager do reblogs = 2.times.map { Fabricate(:status, reblog: reblogged) } # The first reblog will be accepted - FeedManager.instance.push('type', account, reblogs.first) + FeedManager.instance.push_to_home(account, reblogs.first) # Fill the feed with intervening statuses FeedManager::REBLOG_FALLOFF.times do - FeedManager.instance.push('type', account, Fabricate(:status)) + FeedManager.instance.push_to_home(account, Fabricate(:status)) end # The second reblog should also be accepted - expect(FeedManager.instance.push('type', account, reblogs.last)).to be true + expect(FeedManager.instance.push_to_home(account, reblogs.last)).to be true end end end @@ -253,11 +243,11 @@ RSpec.describe FeedManager do reblogged = Fabricate(:status) status = Fabricate(:status, reblog: reblogged) another_status = Fabricate(:status, reblog: reblogged) - reblogs_key = FeedManager.instance.key('type', receiver.id, 'reblogs') - reblog_set_key = FeedManager.instance.key('type', receiver.id, "reblogs:#{reblogged.id}") + reblogs_key = FeedManager.instance.key('home', receiver.id, 'reblogs') + reblog_set_key = FeedManager.instance.key('home', receiver.id, "reblogs:#{reblogged.id}") - FeedManager.instance.push('type', receiver, status) - FeedManager.instance.push('type', receiver, another_status) + FeedManager.instance.push_to_home(receiver, status) + FeedManager.instance.push_to_home(receiver, another_status) # We should have a tracking set and an entry in reblogs. expect(Redis.current.exists(reblog_set_key)).to be true @@ -265,12 +255,12 @@ RSpec.describe FeedManager do # Push everything off the end of the feed. FeedManager::MAX_ITEMS.times do - FeedManager.instance.push('type', receiver, Fabricate(:status)) + FeedManager.instance.push_to_home(receiver, Fabricate(:status)) end # `trim` should be called automatically, but do it anyway, as # we're testing `trim`, not side effects of `push`. - FeedManager.instance.trim('type', receiver.id) + FeedManager.instance.trim('home', receiver.id) # We should not have any reblog tracking data. expect(Redis.current.exists(reblog_set_key)).to be false @@ -285,32 +275,32 @@ RSpec.describe FeedManager do reblogged = Fabricate(:status) status = Fabricate(:status, reblog: reblogged) - FeedManager.instance.push('type', receiver, reblogged) - FeedManager::REBLOG_FALLOFF.times { FeedManager.instance.push('type', receiver, Fabricate(:status)) } - FeedManager.instance.push('type', receiver, status) + FeedManager.instance.push_to_home(receiver, reblogged) + FeedManager::REBLOG_FALLOFF.times { FeedManager.instance.push_to_home(receiver, Fabricate(:status)) } + FeedManager.instance.push_to_home(receiver, status) # The reblogging status should show up under normal conditions. - expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to include(status.id.to_s) + expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to include(status.id.to_s) - FeedManager.instance.unpush('type', receiver, status) + FeedManager.instance.unpush_from_home(receiver, status) # Restore original status - expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to_not include(status.id.to_s) - expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to include(reblogged.id.to_s) + expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to_not include(status.id.to_s) + expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to include(reblogged.id.to_s) end it 'removes a reblogged status if it was only reblogged once' do reblogged = Fabricate(:status) status = Fabricate(:status, reblog: reblogged) - FeedManager.instance.push('type', receiver, status) + FeedManager.instance.push_to_home(receiver, status) # The reblogging status should show up under normal conditions. - expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to eq [status.id.to_s] + expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to eq [status.id.to_s] - FeedManager.instance.unpush('type', receiver, status) + FeedManager.instance.unpush_from_home(receiver, status) - expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to be_empty + expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to be_empty end it 'leaves a multiply-reblogged status if another reblog was in feed' do @@ -318,26 +308,26 @@ RSpec.describe FeedManager do reblogs = 3.times.map { Fabricate(:status, reblog: reblogged) } reblogs.each do |reblog| - FeedManager.instance.push('type', receiver, reblog) + FeedManager.instance.push_to_home(receiver, reblog) end # The reblogging status should show up under normal conditions. - expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to eq [reblogs.first.id.to_s] + expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to eq [reblogs.first.id.to_s] reblogs[0...-1].each do |reblog| - FeedManager.instance.unpush('type', receiver, reblog) + FeedManager.instance.unpush_from_home(receiver, reblog) end - expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to eq [reblogs.last.id.to_s] + expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to eq [reblogs.last.id.to_s] end it 'sends push updates' do status = Fabricate(:status) - FeedManager.instance.push('type', receiver, status) + FeedManager.instance.push_to_home(receiver, status) allow(Redis.current).to receive_messages(publish: nil) - FeedManager.instance.unpush('type', receiver, status) + FeedManager.instance.unpush_from_home(receiver, status) deletion = Oj.dump(event: :delete, payload: status.id.to_s) expect(Redis.current).to have_received(:publish).with("timeline:#{receiver.id}", deletion) diff --git a/spec/models/account_moderation_note_spec.rb b/spec/models/account_moderation_note_spec.rb @@ -1,5 +1,5 @@ require 'rails_helper' RSpec.describe AccountModerationNote, type: :model do - pending "add some examples to (or delete) #{__FILE__}" + end diff --git a/spec/models/feed_spec.rb b/spec/models/feed_spec.rb @@ -1,45 +0,0 @@ -require 'rails_helper' - -RSpec.describe Feed, type: :model do - let(:account) { Fabricate(:account) } - - subject { described_class.new(:home, account) } - - describe '#get' do - before do - Fabricate(:status, account: account, id: 1) - Fabricate(:status, account: account, id: 2) - Fabricate(:status, account: account, id: 3) - Fabricate(:status, account: account, id: 10) - end - - context 'when feed is generated' do - before do - Redis.current.zadd( - FeedManager.instance.key(:home, account.id), - [[4, 4], [3, 3], [2, 2], [1, 1]] - ) - end - - it 'gets statuses with ids in the range from redis' do - results = subject.get(3) - - expect(results.map(&:id)).to eq [3, 2] - expect(results.first.attributes.keys).to eq %w(id updated_at) - end - end - - context 'when feed is being generated' do - before do - Redis.current.set("account:#{account.id}:regeneration", true) - end - - it 'gets statuses with ids in the range from database' do - results = subject.get(3) - - expect(results.map(&:id)).to eq [10, 3, 2] - expect(results.first.attributes.keys).to include('id', 'updated_at') - end - end - end -end diff --git a/spec/models/home_feed_spec.rb b/spec/models/home_feed_spec.rb @@ -0,0 +1,45 @@ +require 'rails_helper' + +RSpec.describe HomeFeed, type: :model do + let(:account) { Fabricate(:account) } + + subject { described_class.new(account) } + + describe '#get' do + before do + Fabricate(:status, account: account, id: 1) + Fabricate(:status, account: account, id: 2) + Fabricate(:status, account: account, id: 3) + Fabricate(:status, account: account, id: 10) + end + + context 'when feed is generated' do + before do + Redis.current.zadd( + FeedManager.instance.key(:home, account.id), + [[4, 4], [3, 3], [2, 2], [1, 1]] + ) + end + + it 'gets statuses with ids in the range from redis' do + results = subject.get(3) + + expect(results.map(&:id)).to eq [3, 2] + expect(results.first.attributes.keys).to eq %w(id updated_at) + end + end + + context 'when feed is being generated' do + before do + Redis.current.set("account:#{account.id}:regeneration", true) + end + + it 'gets statuses with ids in the range from database' do + results = subject.get(3) + + expect(results.map(&:id)).to eq [10, 3, 2] + expect(results.first.attributes.keys).to include('id', 'updated_at') + end + end + end +end diff --git a/spec/models/list_account_spec.rb b/spec/models/list_account_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe ListAccount, type: :model do + +end diff --git a/spec/models/list_spec.rb b/spec/models/list_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe List, type: :model do + +end diff --git a/spec/services/after_block_service_spec.rb b/spec/services/after_block_service_spec.rb @@ -18,8 +18,8 @@ RSpec.describe AfterBlockService do end it "clears account's statuses" do - FeedManager.instance.push(:home, account, status) - FeedManager.instance.push(:home, account, other_account_status) + FeedManager.instance.push_to_home(account, status) + FeedManager.instance.push_to_home(account, other_account_status) is_expected.to change { Redis.current.zrange(home_timeline_key, 0, -1) diff --git a/spec/services/batched_remove_status_service_spec.rb b/spec/services/batched_remove_status_service_spec.rb @@ -30,11 +30,11 @@ RSpec.describe BatchedRemoveStatusService do end it 'removes statuses from author\'s home feed' do - expect(Feed.new(:home, alice).get(10)).to_not include([status1.id, status2.id]) + expect(HomeFeed.new(alice).get(10)).to_not include([status1.id, status2.id]) end it 'removes statuses from local follower\'s home feed' do - expect(Feed.new(:home, jeff).get(10)).to_not include([status1.id, status2.id]) + expect(HomeFeed.new(jeff).get(10)).to_not include([status1.id, status2.id]) end it 'notifies streaming API of followers' do diff --git a/spec/services/fan_out_on_write_service_spec.rb b/spec/services/fan_out_on_write_service_spec.rb @@ -19,12 +19,12 @@ RSpec.describe FanOutOnWriteService do end it 'delivers status to home timeline' do - expect(Feed.new(:home, author).get(10).map(&:id)).to include status.id + expect(HomeFeed.new(author).get(10).map(&:id)).to include status.id end it 'delivers status to local followers' do pending 'some sort of problem in test environment causes this to sometimes fail' - expect(Feed.new(:home, follower).get(10).map(&:id)).to include status.id + expect(HomeFeed.new(follower).get(10).map(&:id)).to include status.id end it 'delivers status to hashtag' do diff --git a/spec/services/mute_service_spec.rb b/spec/services/mute_service_spec.rb @@ -18,8 +18,8 @@ RSpec.describe MuteService do end it "clears account's statuses" do - FeedManager.instance.push(:home, account, status) - FeedManager.instance.push(:home, account, other_account_status) + FeedManager.instance.push_to_home(account, status) + FeedManager.instance.push_to_home(account, other_account_status) is_expected.to change { Redis.current.zrange(home_timeline_key, 0, -1) diff --git a/spec/services/remove_status_service_spec.rb b/spec/services/remove_status_service_spec.rb @@ -25,11 +25,11 @@ RSpec.describe RemoveStatusService do end it 'removes status from author\'s home feed' do - expect(Feed.new(:home, alice).get(10)).to_not include(@status.id) + expect(HomeFeed.new(alice).get(10)).to_not include(@status.id) end it 'removes status from local follower\'s home feed' do - expect(Feed.new(:home, jeff).get(10)).to_not include(@status.id) + expect(HomeFeed.new(jeff).get(10)).to_not include(@status.id) end it 'sends PuSH update to PuSH subscribers' do diff --git a/spec/workers/feed_insert_worker_spec.rb b/spec/workers/feed_insert_worker_spec.rb @@ -11,41 +11,41 @@ describe FeedInsertWorker do context 'when there are no records' do it 'skips push with missing status' do - instance = double(push: nil) + instance = double(push_to_home: nil) allow(FeedManager).to receive(:instance).and_return(instance) result = subject.perform(nil, follower.id) expect(result).to eq true - expect(instance).not_to have_received(:push) + expect(instance).not_to have_received(:push_to_home) end it 'skips push with missing account' do - instance = double(push: nil) + instance = double(push_to_home: nil) allow(FeedManager).to receive(:instance).and_return(instance) result = subject.perform(status.id, nil) expect(result).to eq true - expect(instance).not_to have_received(:push) + expect(instance).not_to have_received(:push_to_home) end end context 'when there are real records' do it 'skips the push when there is a filter' do - instance = double(push: nil, filter?: true) + instance = double(push_to_home: nil, filter?: true) allow(FeedManager).to receive(:instance).and_return(instance) result = subject.perform(status.id, follower.id) expect(result).to be_nil - expect(instance).not_to have_received(:push) + expect(instance).not_to have_received(:push_to_home) end it 'pushes the status onto the home timeline without filter' do - instance = double(push: nil, filter?: false) + instance = double(push_to_home: nil, filter?: false) allow(FeedManager).to receive(:instance).and_return(instance) result = subject.perform(status.id, follower.id) expect(result).to be_nil - expect(instance).to have_received(:push).with(:home, follower, status) + expect(instance).to have_received(:push_to_home).with(follower, status) end end end diff --git a/streaming/index.js b/streaming/index.js @@ -254,6 +254,26 @@ const startWorker = (workerId) => { const placeholders = (arr, shift = 0) => arr.map((_, i) => `$${i + 1 + shift}`).join(', '); + const authorizeListAccess = (id, req, next) => { + pgPool.connect((err, client, done) => { + if (err) { + next(false); + return; + } + + client.query('SELECT id, account_id FROM lists WHERE id = $1 LIMIT 1', [id], (err, result) => { + done(); + + if (err || result.rows.length === 0 || result.rows[0].account_id !== req.accountId) { + next(false); + return; + } + + next(true); + }); + }); + }; + const streamFrom = (id, req, output, attachCloseHandler, needsFiltering = false, notificationOnly = false) => { const streamType = notificationOnly ? ' (notification)' : ''; log.verbose(req.requestId, `Starting stream from ${id} for ${req.accountId}${streamType}`); @@ -410,7 +430,22 @@ const startWorker = (workerId) => { streamFrom(`timeline:hashtag:${req.query.tag.toLowerCase()}:local`, req, streamToHttp(req, res), streamHttpEnd(req), true); }); - const wss = new WebSocket.Server({ server, verifyClient: wsVerifyClient }); + app.get('/api/v1/streaming/list', (req, res) => { + const listId = req.query.list; + + authorizeListAccess(listId, req, authorized => { + if (!authorized) { + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Not found' })); + return; + } + + const channel = `timeline:list:${listId}`; + streamFrom(channel, req, streamToHttp(req, res), streamHttpEnd(req, subscriptionHeartbeat(channel))); + }); + }); + + const wss = new WebSocket.Server({ server, verifyClient: wsVerifyClient }); wss.on('connection', ws => { const req = ws.upgradeReq; @@ -443,6 +478,19 @@ const startWorker = (workerId) => { case 'hashtag:local': streamFrom(`timeline:hashtag:${location.query.tag.toLowerCase()}:local`, req, streamToWs(req, ws), streamWsEnd(req, ws), true); break; + case 'list': + const listId = location.query.list; + + authorizeListAccess(listId, req, authorized => { + if (!authorized) { + ws.close(); + return; + } + + const channel = `timeline:list:${listId}`; + streamFrom(channel, req, streamToWs(req, ws), streamWsEnd(req, ws, subscriptionHeartbeat(channel))); + }); + break; default: ws.close(); }