Who has most votes on answers and questions?

I’m trying to get the number of votes on both questions (topics) and answers (posts). This is close, but I can’t quite join the data across both subqueries. Any SQL gurus out there?

WITH topic_votes AS (
SELECT sum (to_number(tcf.value, '9999')) as total_topic_votes, 
    u.id user_id
    FROM users u
LEFT JOIN topics t
ON u.id=t.user_id
JOIN topic_custom_fields tcf
ON t.id=tcf.topic_id
WHERE tcf.name='vote_count'
GROUP by u.id
),
post_votes AS (
SELECT 
  SUM(to_number(pcf.value, '9999')) as total_post_votes,
  u.id user_id
  FROM users u
  LEFT JOIN posts p
  ON u.id=p.user_id
  JOIN post_custom_fields pcf
  on p.id=pcf.post_id
  WHERE pcf.name='vote_count'
  GROUP BY u.id
)

SELECT coalesce(sum(total_topic_votes),0) tv,
       coalesce(sum(total_post_votes),0) pv,
       post_votes.user_id user_id,
       sum(total_topic_votes) + sum(total_post_votes)  as total
       from post_votes 
       full outer join topic_votes
       on post_votes.user_id=topic_votes.user_id
       group by post_votes.user_id,topic_votes.user_id
       order by total desc