## Code for reproducing figures from Hawkins and Silva:

library(ggplot2)
library(extrafont)
loadfonts()
library(Hmisc)
library(irr)
library(gridExtra)

manif<-read.csv('scores_coding_manifestos_CHES2014_updateApril18.csv')

## Clean data just in case Excel screwed up smth:
manif[manif == '#DIV/0!']<-NA
manif[,c(4:(ncol(manif)-3))]<-sapply(manif[,c(4:(ncol(manif)-3))],as.character)
manif[,c(4:(ncol(manif)-3))]<-sapply(manif[,c(4:(ncol(manif)-3))],as.numeric)

manif<-subset(manif, Country != 'Australia')

manif$logmanif<-log(manif$Manifesto_total*10 + 1)
manif$logspeech<-log(manif$Speeches_total*10 + 1)
manif$logparty<-log(manif$Party_total*10 + 1)
manif$logparty2<-log(manif$Party_total_2*10+1)

### T-test comparing preambles and list of positions:

t.test(manif$Issues_mean,manif$Preamble_mean)

### T-test comparing manifestos means and speeches means:
t.test(manif$Manifesto_total,manif$Speeches_total)

#### Speech x manifestos graph (FIGURE 1 PAPER):

sum(manif$Party_total[manif$LatAm == 0]>=0,na.rm=T)

sp<-subset(manif,Speeches_total != 'NA' & Manifesto_total != 'NA')
sp$LatAm[sp$euro == 0]<-'Americas'
sp$LatAm[sp$euro == 1]<-'Europe'

cor_sp_man<-round(cor(sp$Speeches_total,sp$Manifesto_total,use='pairwise.complete.obs'),2)


p<-ggplot(data = sp, aes(x = Speeches_total, y = Manifesto_total, label = Party,colour = factor(LatAm)))+geom_text()+
  labs(y = 'Party manifestos', x = paste('Candidates speeches \n \n\ r = ',cor_sp_man,', n = ', nrow(sp),sep='')) +
  scale_color_manual(values = c('black','gray50')) + ylim(c(-0.2,2.2))+xlim(c(-0.2,2.2))+
  theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(), 
        panel.background = element_blank(),axis.line.x = element_line(colour = 'black'),
        axis.line.y = element_line(colour = 'black'),
        legend.key = element_blank(),legend.title = element_blank(),axis.text.x = element_text(colour = 'black'),axis.text.y = element_text(colour = 'black'),legend.justification=c(0,1),legend.position=c(0,1),
        text=element_text(family='Gill Sans'))
p

ggsave('Latex_manuscript/graphs/speech_x_manif_final.png', p, device='png', dpi=1200, width=5, height=5)


#### FIGURE 2, INTERCODER RELIABILITY:
manif2<-subset(manif, Country != 'France')

coder1.m<-matrix(ncol=1,c(manif2$Coder1_pre,manif2$Coder1_iss,manif2$Manif_coder1_total[is.na(manif2$Coder1_iss)]),byrow=T)
coder1.sp<-matrix(ncol=1,c(manif2$Speech1_coder1, manif2$Speech2_coder1))
# coder1<-rbind(coder1,coder1.sp)

coder2.m<-matrix(ncol=1,c(manif2$Coder2_pre,manif2$Coder2_iss,manif2$Manif_coder2_total[is.na(manif2$Coder1_iss)]),byrow=T) ## Keep is.na with coder1 to make sure that you're not removing those that are NA because there was just 1 coder...
coder2.sp<-matrix(ncol=1,c(manif2$Speech1_coder2, manif2$Speech2_coder2))
# coder2<-rbind(coder2, coder2.sp)

grades.m<-cbind(coder1.m,coder2.m)
grades.m<-na.omit(grades.m)
grades.m<-data.frame(grades.m)

kalpha.m<-kripp.alpha(t(grades.m[,1:2]),method='interval')

grades.sp<-cbind(coder1.sp,coder2.sp)
grades.sp<-na.omit(grades.sp)
grades.sp<-data.frame(grades.sp)

kalpha.sp<-kripp.alpha(t(grades.sp[,1:2]),method='interval')

p3.1<-ggplot(data = grades.m, aes(x = X1, y = X2))+
  geom_count() +
  labs(size = '# of \n Texts',y = 'Coder 1', x = paste("Coder 2 \n \n Krippendorff's alpha = ", round(kalpha.m$value,2)," n = ",kalpha.m$subjects,sep=''),
       title = 'Manifestos')+
  theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(), 
        panel.background = element_blank(),axis.line.x = element_line(colour = 'black'),
        axis.line.y = element_line(colour = 'black'),axis.text = element_text(colour = 'black'),
        legend.key = element_blank(),
        #legend.title = element_text(),
        text=element_text(family = 'Gill Sans'))
p3.1

p3.2<-ggplot(data = grades.sp, aes(x = X1, y = X2))+
  geom_count() +
  labs(size = '# of \n Texts',y = 'Coder 1', x = paste("Coder 2 \n \n Krippendorff's alpha = ", round(kalpha.sp$value,2)," n = ",kalpha.sp$subjects,sep=''),
       title = 'Speeches')+scale_size_continuous(breaks = c(2,4,8))+ 
  theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(), 
        panel.background = element_blank(),axis.line.x = element_line(colour = 'black'),
        axis.line.y = element_line(colour = 'black'),axis.text = element_text(colour = 'black'),
        legend.key = element_blank(),
        text=element_text(family='Gill Sans'))
p3.2

pp<-grid.arrange(p3.1,p3.2,ncol=2)


ggsave('Latex_manuscript/graphs/coder_reliability_final.png', pp, device='png', dpi=1200, width=8, height=5)


### FIGURE 3: POPULISM BY REGION:

a<-NULL
a<-data.frame(rbind(mean(manif$Manifesto_total[manif$euro == 1],na.rm=T),
                    mean(manif$Manifesto_total[manif$LatAm == 1],na.rm=T),
                    wtd.mean(manif$Manifesto_total[manif$euro == 1],manif$elec_result_major[manif$euro == 1]),
                    wtd.mean(manif$Manifesto_total[manif$LatAm == 1],manif$elec_result_major[manif$LatAm == 1])))
colnames(a)<-'Manifesto.scores'
a$region<-c('Europe','Latin America','Europe','Latin America')
a$calc<-c('Mean','Mean','Weighed mean','Weighed mean')
p1<-ggplot(data = a, aes(x = region, y = Manifesto.scores, fill = calc))+
  geom_bar(stat='identity',position = 'dodge')+
  labs(title = 'Manifestos', y = 'Mean scores') +ylim(c(0, 0.6))+
  scale_fill_manual(values = c('grey50','black'))+theme(panel.grid.major = element_blank(),
                                                            panel.grid.minor = element_blank(), panel.background = element_blank(),
                                                            axis.text.x = element_text(colour = 'black'),axis.text.y = element_text(colour = 'black'),
                                                            axis.line.x = element_line(colour = 'black'),axis.line.y = element_line(colour = 'black'),legend.key = element_blank(),axis.title.x = element_blank(),
                                                            legend.title = element_blank(),legend.position='bottom',
                                                        text=element_text(family='Gill Sans'))
p1


ggsave('Latex_manuscript/graphs/region_means.png', p1, device='png', dpi=1200, height=5, width=5)


### Video vs. transcripts of speeches:
sp1<-data.frame(cbind(manif$Speech1_total,manif$Speech1_transcript,manif$elec_result_major,manif$Manifesto_total))
sp2<-data.frame(cbind(manif$Speech2_total,manif$Speech2_transcript,manif$elec_result_major,manif$Manifesto_total))
sp<-rbind(sp1,sp2)

## Comparison of mean speech scores:
t.test(sp$X1~sp$X2)

## comparison of electoral results based on kind of document:
t.test(sp$X3~sp$X2)

## comparison of mean manifesto scores:
t.test(sp$X4~sp$X2)

### Automated analysis:

rm(list=ls())

library(tm)
library(ggplot2)
library(dplyr)
library(irr)
library(glmnet)

set.seed(123)

grades<-read.csv('manifestos.csv')

## Get documents: 
cname<-file.path('converted2')
cname.s<-file.path('Translated')
docs<-Corpus(DirSource(c(cname,cname.s)))

## Remove punctuation, numbers, converting to lowercase, removing stopwords, stemming:
docs<-tm_map(docs, removeNumbers)
docs<-tm_map(docs, content_transformer(tolower))
docs<-tm_map(docs, removeWords, stopwords('en'))
docs<-tm_map(docs, removePunctuation)
docs<-tm_map(docs, stemDocument) ### If returning mclapply error, install package SnowballC manually
docs <- tm_map(docs, stripWhitespace)   

## Create the document term matrix
dtm.all<-DocumentTermMatrix(docs)
rm(docs)

## Recode the grades:

# Binary outcome:
grades$bi<-0
grades$bi[grades$grade >= 0.5]<-1

# Original three categories:
grades$original<-1
grades$original[grades$grade < 0.5]<-0
grades$original[grades$grade >= 1.5]<-2

## Function to remove common terms
removeCommonTerms <- function (x, pct) 
{
  stopifnot(inherits(x, c("DocumentTermMatrix", "TermDocumentMatrix")), 
            is.numeric(pct), pct > 0, pct < 1)
  m <- if (inherits(x, "DocumentTermMatrix")) 
    t(x)
  else x
  t <- table(m$i) < m$ncol * (pct)
  termIndex <- as.numeric(names(t[t]))
  if (inherits(x, "DocumentTermMatrix")) 
    x[, termIndex]
  else x[termIndex, ]
}

### Manifestos:
### Test different levels of lambda, alpha, and removal of sparse and common terms through LOOCV:
dtm<-dtm.all[1:154,]
alphas<-seq(0,1,.1)
common<-seq(0.7, 0.95, 0.05)
sparse<-seq(0.7,0.95,0.05)
results<-matrix(NA,nrow=1,ncol=6)
for(j in 1:6){
  cm<-common[j]
  for(k in 1:6){
    sp<-sparse[k]
    mat1<-removeSparseTerms(dtm,sp)
    mat1<-removeCommonTerms(mat1,cm)
    mat1<-as.matrix(mat1)
    for(i in 1:11){
      fit.cv<-cv.glmnet(x=mat1,y=grades$bi,type.measure='class',nfolds=nrow(mat1),family='binomial',alpha=alphas[i])
      values<-cbind(i, j, k, fit.cv$lambda.min, min(fit.cv$cvm), fit.cv$cvsd[which.min(fit.cv$cvm)])
      results<-rbind(results,values)
    }
  }
}


which.min(results[,5])
results[73,]

## Alpha: 0.5
## Common: 0.75
## Sparse: 0.7
## Lambda: 0.008060196 ## This has slight variations each time. Need to set seed
## Min cvm: 0.123376623 ## Ditto

mat1<-removeSparseTerms(dtm,0.7)
mat1<-removeCommonTerms(mat1,0.75)
mat1<-as.matrix(mat1)
fit.cv.final<-cv.glmnet(x=mat1,y=grades$bi,type.measure='class',nfolds=nrow(mat1),family='binomial',alpha=0.5,keep=T)

# Get predicted values:
fit.values<-predict(fit.cv.final,mat1,type='class')

# Table:
table(fit.values,grades$bi)
# Kalpha:
kripp.alpha(t(cbind(fit.values,grades$bi)), method='nominal')


### Original scores, manifestos:
results.o<-matrix(NA,nrow=1,ncol=6)
for(j in 1:6){
  cm<-common[j]
  for(k in 1:6){
    sp<-sparse[k]
    mat1<-removeSparseTerms(dtm,sp)
    mat1<-removeCommonTerms(mat1,cm)
    mat1<-as.matrix(mat1)
for(i in 1:11){
  fit.cv.o<-cv.glmnet(x=mat1,y=grades$orig,type.measure='class',nfolds=nrow(mat1),family='multinomial',alpha=alphas[i])
  values.o<-cbind(i, j, k, fit.cv.o$lambda.min, min(fit.cv.o$cvm), fit.cv.o$cvsd[which.min(fit.cv.o$cvm)])
  results.o<-rbind(results.o,values.o)
}
  }
}
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          
## Find out better combination of parameters:
results[which.min(results.o[,5]),]
# Better alpha: 0.9

fit.cv.final.o<-cv.glmnet(x=mat1,y=grades$orig,type.measure='class',nfolds=nrow(mat1),family='multinomial',alpha=0.9,keep=T)
fit.values.o<-predict(fit.cv.final.o,mat1,type='class')

table(fit.values.o,grades$orig)

kripp.alpha(t(cbind(fit.values.o,grades$orig)), method='ordinal')


### Speeches:

grades.speeches<-read.csv('speeches.grades.csv')

grades.speeches$bi<-0
grades.speeches$bi[grades.speeches$score >= 0.5]<-1

# Original three categories:
grades.speeches$original<-1
grades.speeches$original[grades.speeches$score < 0.5]<-0
grades.speeches$original[grades.speeches$score >= 1.5]<-2

dtm.s<-dtm.all[155:226,]
mat.s<-as.matrix(dtm.s)
results.s<-matrix(NA,nrow=1,ncol=4)
for(i in 1:11){
  fit.cv.s<-cv.glmnet(x=mat.s,y=grades.speeches$bi,type.measure='class',nfolds=nrow(mat.s),family='binomial',alpha=alphas[i])
  values.s<-cbind(i,fit.cv.s$lambda.min, min(fit.cv.s$cvm), fit.cv.s$cvsd[which.min(fit.cv.s$cvm)])
  results.s<-rbind(results.s,values.s)
}

## select better alpha,
which.min(results.s[,3])
# Better alpha: 0.9

fit.cv.final.s<-cv.glmnet(x=mat.s,y=grades.speeches$bi,type.measure='class',nfolds=nrow(mat.s),family='binomial',alpha=0.6,keep=T)
fit.values.s<-predict(fit.cv.final.s,mat.s,type='class')

table(fit.values.s,grades.speeches$bi)

kripp.alpha(t(cbind(fit.values.s,grades.speeches$bi)), method='nominal')



#### Figure 4: Stems from the binary coding of manifestos:
#### Stems from the binary classification:
small.lambda.index <- which(fit.cv.final$lambda == fit.cv.final$lambda.min)
small.lambda.betas <- fit.cv.final$glmnet.fit$beta[,small.lambda.index]
coefs.pop<-as.matrix(small.lambda.betas[small.lambda.betas > 0])
coefs.npop<-as.matrix(small.lambda.betas[small.lambda.betas < 0])
coefs<-data.frame(rbind(coefs.pop,coefs.npop))
coefs$stems<-rownames(coefs)
colnames(coefs)<-c('coef','stems')
coefs$random<-runif(nrow(coefs),min=0,max=1)
coefs$pos<-ifelse(coefs$coef > 0,1,0)
p<-ggplot(data=coefs,aes(x = coef,y=random))+geom_text(aes(label=stems,size=abs(coef),colour=abs(coef)))+
  scale_colour_gradient(low='grey60', high='black',guide=FALSE)+xlim(c(-0.4,0.6))+xlab('Coefficients')+
  scale_size(guide=FALSE)+
  theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
        axis.text.x = element_text(colour = 'black'),axis.text.y = element_blank(),
        axis.title.y = element_blank(),axis.ticks.y = element_blank(),
        panel.background = element_blank(),axis.line.x = element_line(colour = 'black'),
        legend.key = element_blank(),legend.title = element_blank(),
        text=element_text(family='Gill Sans'))
p

ggsave('Latex_manuscript/graphs/stems_binary_final.png', p, device='png', dpi=1200, width=6, height=6)


pdf('Latex_manuscript/graphs/stems_binary.pdf',height=6, width=5)
p
dev.off()
embed_fonts('Latex_manuscript/graphs/stems_binary.pdf')

