2020_09_29_taylor_swift_beyonce.Rmd

---
title: "TidyTemplate"
date: 2020-09-29
output: html_output
---

# TidyTuesday

```{r setup, include=FALSE}

knitr::opts_chunk$set(echo = TRUE)

library(tidyverse)
library(tidytuesdayR)
library(scales)
library(tidytext)
library(lubridate)
library(tidylo)
theme_set(theme_light())
```

# Load the weekly Data

Dowload the weekly data and make available in the `tt` object.

```{r Load}
beyonce_lyrics <- readr::read_csv('https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2020/2020-09-29/beyonce_lyrics.csv')
taylor_swift_lyrics <- readr::read_csv('https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2020/2020-09-29/taylor_swift_lyrics.csv')
sales <- readr::read_csv('https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2020/2020-09-29/sales.csv')
charts <- readr::read_csv('https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2020/2020-09-29/charts.csv')
```

```{r}
beyonce_lyrics %>%
  count(song_name, sort = TRUE)

taylor_swift_lyrics %>%
  count(Title, sort = TRUE)

taylor_swift_lyrics %>%
  count(Album, sort = TRUE)

beyonce_lyrics %>%
  count(artist_name, sort = TRUE)

charts

sales %>%
  filter(title == "1989")
```

### Looking at sales and charts

```{r}
sales %>%
  filter(country == "US") %>%
  mutate(title = fct_reorder(title, sales)) %>%
  ggplot(aes(sales, title, fill = artist)) +
  geom_col() +
  scale_x_continuous(labels = dollar) +
  labs(x = "Sales (US)",
       y = "")

sales %>%
  filter(country %in% c("World", "WW")) %>%
  mutate(title = fct_reorder(title, sales)) %>%
  ggplot(aes(sales, title, fill = artist)) +
  geom_col() +
  scale_x_continuous(labels = dollar) +
  labs(x = "Sales (World)",
       y = "")

charts %>%
  filter(chart == "US")
```

### Text analysis of Taylor Swift

```{r}
release_dates <- charts %>%
  distinct(album = title, released) %>%
  mutate(album = fct_recode(album,
                            folklore = "Folklore",
                            reputation = "Reputation")) %>%
  mutate(released = str_remove(released, " \\(.*")) %>%
  mutate(released = mdy(released))

taylor_swift_words <- taylor_swift_lyrics %>%
  rename_all(str_to_lower) %>%
  select(-artist) %>%
  unnest_tokens(word, lyrics) %>%
  anti_join(stop_words, by = "word") %>%
  inner_join(release_dates, by = "album") %>%
  mutate(album = fct_reorder(album, released))
```

```{r}
taylor_swift_words %>%
  count(word, sort = TRUE) %>%
  head(25) %>%
  mutate(word = fct_reorder(word, n)) %>%
  ggplot(aes(n, word)) +
  geom_col()
```

```{r}
ts_tf_idf <- taylor_swift_words %>%
  count(album, word) %>%
  bind_tf_idf(word, album, n) %>%
  arrange(desc(tf_idf))

ts_tf_idf %>%
  group_by(album) %>%
  slice_max(tf_idf, n = 10, with_ties = FALSE) %>%
  ungroup() %>%
  mutate(word = reorder_within(word, tf_idf, album)) %>%
  ggplot(aes(tf_idf, word)) +
  geom_col() +
  facet_wrap(~ album, scales = "free_y") +
  scale_y_reordered()
```

```{r}
ts_lo <- taylor_swift_words %>%
  count(album, word) %>%
  bind_log_odds(album, word, n) %>%
  arrange(desc(log_odds_weighted))

ts_lo %>%
  group_by(album) %>%
  slice_max(log_odds_weighted, n = 10, with_ties = FALSE) %>%
  ungroup() %>%
  mutate(word = reorder_within(word, log_odds_weighted, album)) %>%
  ggplot(aes(log_odds_weighted, word)) +
  geom_col() +
  facet_wrap(~ album, scales = "free_y") +
  scale_y_reordered()

filler <- c("ah", "uh", "ha", "ey", "eh", "eeh", "huh")

ts_lo %>%
  filter(word %in% filler) %>%
  mutate(word = reorder_within(word, n, album)) %>%
  ggplot(aes(n, word)) +
  geom_col() +
  facet_wrap(~ album, scales = "free_y") +
  scale_y_reordered() +
  labs(title = "The filler words in Taylor Swift lyrics have changed across albums",
       x = "# of appearances in album",
       y = "")
```

```{r}
ts <- taylor_swift_lyrics %>%
  rename_all(str_to_lower) %>%
  rename(song = title) %>%
  select(-album)

beyonce <- beyonce_lyrics %>%
  select(artist = artist_name, song = song_name, lyrics = line)

artist_song_words_raw <- bind_rows(ts, beyonce) %>%
  unnest_tokens(word, lyrics) %>%
  count(artist, song, word)

artist_song_words <- artist_song_words_raw %>%
  anti_join(stop_words, by = "word")
```

# Compare overall, not per song

```{r}
by_artist_word <- artist_song_words %>%
  group_by(artist, word) %>%
  summarize(num_songs = n(),
            num_words = sum(n)) %>%
  mutate(pct_words = num_words / sum(num_words)) %>%
  group_by(word) %>%
  mutate(num_words_total = sum(num_words)) %>%
  ungroup()

word_differences <- by_artist_word %>%
  bind_log_odds(artist, word, num_words) %>%
  arrange(desc(abs(log_odds_weighted))) %>%
  filter(artist == "Beyoncé") %>%
  slice_max(num_words_total, n = 100, with_ties = FALSE) %>%
  slice_max(abs(log_odds_weighted), n = 25, with_ties = FALSE) %>%
  mutate(word = fct_reorder(word, log_odds_weighted)) %>%
  mutate(direction = ifelse(log_odds_weighted > 0, "Beyoncé", "Taylor Swift"))

word_differences %>%
  ggplot(aes(log_odds_weighted, word, fill = direction)) +
  geom_col() +
  scale_x_continuous(breaks = log(2 ^ seq(-6, 9, 3)),
                     labels = paste0(2 ^ abs(seq(-6, 9, 3)), "X")) +
  labs(x = "Relative use in Beyoncé vs Taylor Swift (weighted)",
       y = "",
       title = "Which words most distinguish Beyoncé and Taylor Swift songs?",
       subtitle = "Among the 100 words most used by the artists (combined)",
       fill = "")

x_labels <- paste0(2 ^ abs(seq(-6, 9, 3)), "X")
x_labels <- ifelse(x_labels == "1X", "Same", x_labels)

word_differences %>%
  ggplot(aes(log_odds_weighted, word)) +
  geom_col(width = .1) +
  geom_point(aes(size = num_words_total, color = direction)) +
  geom_vline(lty = 2, xintercept = 0) +
  scale_x_continuous(breaks = log(2 ^ seq(-6, 9, 3)),
                     labels = x_labels) +
  labs(x = "Relative use in Beyoncé vs Taylor Swift (weighted)",
       y = "",
       title = "Which words most distinguish Beyoncé and Taylor Swift songs?",
       subtitle = "Among the 100 words most used by the artists (combined)",
       color = "",
       size = "# of words\n(both artists)")
```

```{r}
comparison <- by_artist_word %>%
  select(artist, word, pct_words, num_words_total) %>%
  pivot_wider(names_from = artist,
              values_from = pct_words,
              values_fill = list(pct_words = 0)) %>%
  janitor::clean_names() %>%
  slice_max(num_words_total, n = 200, with_ties = FALSE)

comparison %>%
  ggplot(aes(taylor_swift, beyonce)) +
  geom_abline(color = "red") +
  geom_point() +
  geom_text(aes(label = word), vjust = 1, hjust = 1, check_overlap = TRUE) +
  scale_x_log10(labels = percent) +
  scale_y_log10(labels = percent)

comparison %>%
  ggplot(aes(num_words_total, beyonce / taylor_swift)) +
  geom_hline(yintercept = 1) +
  geom_point() +
  geom_text(aes(label = word), vjust = 1, hjust = 1, check_overlap = TRUE) +
  scale_x_log10(labels = percent) +
  scale_y_log10()
```

We could have done:

* Sentiment analysis: who is happier, what albums are happier?
* Supervised learning: can you distinguish a Taylor Swift song from a Beyoncé song?