How to manually change text color of ggplot2 legen

There might be a simple way to do this, but I am not sure what it is. I am trying to make it so that the text in the legend matches up with the color box next to it. I have been trying to do this for a while and have not found a way to use the element_text function to add multiple colors to the legend. I've had no problem making every label the same color, but is there a way to make each legend label a different color?

data<-data.frame(count=c(39,36,19,6), category=c("a","b","c","d"))
data$fraction = data$count / sum(data$count)
data = data[order(data$fraction), ]
data$ymax = cumsum(data$fraction)
data$ymin = c(0, head(data$ymax, n=-1))

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#Create Plot
fill <- c("blue3","cyan3","darkgrey","forestgreen")

library(ggplot2)

p1 = ggplot(data, aes(fill=category, ymax=ymax, ymin=ymin, xmax=4, xmin=3.5)) +  
 geom_rect(colour="White") +
 coord_polar(theta="y") +
 scale_fill_manual(values=fill)+
 theme_bw()+
 geom_label(aes(label=paste(data$fraction*100,"%"),x=4,y=
 (ymin+ymax)/2),inherit.aes = F)+
 theme(panel.grid=element_blank())+
 theme(axis.ticks=element_blank()) +     
 xlim(c(0, 4)) +
 theme(axis.text=element_blank()) +
 theme(legend.text=element_text(color=fill,size=12))+
 theme(legend.key.size=unit(2,'lines'))+
 theme(legend.key=element_rect(size=5))+
 labs(title="donut plot")


 print(p1)

With a couple of modifications to this answer, match-legend-text-color-in-geom-text-to-symbol, you get what you want. But note, the answer uses grid's editing functions.

# Your data and plot
data<-data.frame(count=c(39,36,19,6), category=c("a","b","c","d"))
data$fraction = data$count / sum(data$count)
data = data[order(data$fraction), ]
data$ymax = cumsum(data$fraction)
data$ymin = c(0, head(data$ymax, n=-1))


fill <- c("blue3","cyan3","darkgrey","forestgreen")

library(ggplot2)

p1 = ggplot(data, aes(fill=category, ymax=ymax, ymin=ymin, xmax=4, xmin=3.5)) +
 geom_rect(colour="White") +
 coord_polar(theta="y") +
 scale_fill_manual(values=fill)+
 theme_bw()+
 geom_label(aes(label=paste(data$fraction*100,"%"),x=4,y=
 (ymin+ymax)/2),inherit.aes = F)+
 theme(panel.grid=element_blank())+
 theme(axis.ticks=element_blank()) +     
 xlim(c(0, 4)) +
 theme(axis.text=element_blank()) +
 theme(legend.text=element_text(color=fill,size=12))+
 theme(legend.key.size=unit(2,'lines'))+
 theme(legend.key=element_rect(size=5))+
 labs(title="donut plot")


# Get the ggplot grob
g <- ggplotGrob(p1)

# Check out the grobs
library(grid)
grid.ls(grid.force(g))

Look through the list of grobs. The grobs you want to edit are towards the bottom of the list, in the 'guide-box' set of grobs - with names that begin with "label". There are four grobs:

label-3-3.4-4-4-4
label-4-3.5-4-5-4
label-5-3.6-4-6-4
label-6-3.7-4-7-4

# Get names of 'label' grobs.
names.grobs <- grid.ls(grid.force(g))$name 
labels <- names.grobs[which(grepl("^label", names.grobs))]

# Edit the 'label' grobs - change their colours
# Use the `editGrob` function
for(i in seq_along(labels)) {
    g <- editGrob(grid.force(g), gPath(labels[i]), grep = TRUE,  
         gp = gpar(col = fill[i]))
}

# Draw it
grid.newpage()
grid.draw(g)