--- title: "What is statistics?" author: "Professor Sean van der Merwe" format: revealjs: theme: [default, UFS1.scss] logo: UFSlogo2022SvdM.svg footer: "Young stars" slide-number: "c/t" chalkboard: false parallax-background-image: "AIchat.jpg" parallax-background-size: "2458px 1229px" parallax-background-vertical: 35 execute: echo: false fig-width: 7.5 fig-height: 4.5 --- ```{r, warning=FALSE, message=FALSE, results='hide'} options(scipen = 12) library(knitr) library(tidyverse) library(openxlsx) UFScolours <- c(Blue="#0F204BFF", Red="#A71930FF", LightGray="#A7A8AAFF", DarkGray="#8D8D8EFF", NAS="#0039A7FF", EDU="#00675AFF", THEO="#C69317FF", HUM="#EA8400FF", SA="#A40084FF", SC="#9E83B7FF", QWA="#00B140FF", LAW="#BB133EFF", HLTH="#490E6FFF", Black="#000000FF", VeryLight="#D7D8DDFF") cols <- UFScolours[c(8,11,10,2)] mapvalues <- function(x, from=unique(x), to=seq_along(from), warn_missing=FALSE) { # Simplified replacement for mapvalues from plyr package, by Sean van der Merwe, UFS to <- unlist(to) # Make sure it's a vector. Also note that warn_missing is ignored. to[match(x, from)] } wrap_strings <- function(vector_of_strings, width){ as.character( sapply( vector_of_strings, \(x) { paste( strwrap(x, width=width), collapse="\n")}))} library(plotly) fig_w <- 960 fig_h <- 400 plotly_table_UFS <- function(datafrm, height = "100%", colors=UFScolours, useRowNames = FALSE, rowNamesHeading = " ") { # For the colour palette: the first two and last two colours are used datafrm <- as.data.frame(datafrm) nms <- names(datafrm) if (useRowNames) { datafrm <- data.frame(row.names(datafrm), datafrm, row.names = NULL) nms <- c(rowNamesHeading, nms) names(datafrm) <- nms } k <- ncol(datafrm) algn <- rep('center', k) cell_values <- rbind(t(as.matrix(unname(datafrm)))) fig <- plot_ly( type = 'table', height = height, header = list( values = nms, align = algn, line = list(width = 1, color = colors[2]), fill = list(color = colors[1]), font = list(family = "Arial", size = 14, color = "#FFFFFFFF") ), cells = list( values = cell_values, align = algn, line = list(color = colors[2], width = 1), fill = list(color = list(rep(c("#FFFFFFFF", colors[length(colors)]), ceiling(length(cell_values)/2)))), font = list(family = "Arial", size = 12, color = colors[length(colors)-1]) )) fig } plotly_colours <- UFScolours |> set_names(NULL) ``` # What is statistics? "Statistics is the grammar of science." - A statistic is just a summary of some numbers - So statistics starts with describing patterns in numbers - But numbers are never certain nor exact - So statistics is really about **understanding uncertainty** ## Solving scientific problems ```{r} #| fig-height: 6.6 library(visNetwork) cols <- rev(viridisLite::turbo(8))[-1] nodes <- data.frame(id = 1:10, label = c('Observe', 'Hypothesise', 'Model', 'Experiment', 'Data', 'Equations', 'Code', 'Results', 'Graphs', 'Tables'), shape = c(rep("box", 10)), color = cols[c(1, 2, 2, 3, 3, 4, 4, 5, 5, 5)], x = c(1:10)*80, y = c(rep(0, 8), rep(c(1, -1), 1))*100, physics = rep(c(TRUE, FALSE), c(7, 3)), font = "20px" ) edges <- data.frame(from = c(1:4, 3, 6:8, 8, 5, 5), to = c(2:5, 6, 7:9, 10, 8, 7), arrows = "to", length = c(rep(40, 11)) ) net1 <- visNetwork(nodes, edges, physics = TRUE) library(htmlwidgets) library(widgetframe) frameWidget(net1) ``` > Statisticians get to help with every step! ## Maths as foundation - In our department we do not do maths for the sake of maths (the Maths department does that) - We do maths because it is the foundation of so much more - Actuarial Science is built on Mathematical Statistics - Mathematical Statistics is built on Mathematics # Examples ## Let's shuffle a deck of cards ```{r} #| echo: true deck <- paste( c(2:10, 'Jack', 'Queen', 'King', 'Ace') |> rep(times=4), 'of', c('Spades','Diamonds','Clubs','Hearts') |> rep(each=13) ) # To draw a hand of 7 cards: hand <- deck |> sample(7) # Shuffling is taking a sample the same size (52) without replacement: shuffled_deck <- deck |> sample(length(deck)) # To do a bootstrap sample just add: , TRUE ``` ```{r} data.frame(Deck = deck, Shuffled = shuffled_deck) |> plotly_table_UFS() ``` ## Text information Perhaps we shall summarise the complete works of Shakespeare in a word cloud? ```{r} #| fig-height: 6 #| fig-width: 8 freq_df <- readRDS("Shakespeare_words_above4.rds") library(wordcloud2) freq_df |> wordcloud2() |> frameWidget() ``` ## Survey exploration ```{r} boxplot_plotly <- \() { spending <- expand.grid(Gender = c("Female", "Male"), City = c('Bloemfontein', 'East London', 'Durban'), Replicate = 1:20) set.seed(12345) spending$Weekly_Spending <- round( (rgamma(nrow(spending), 10 + (spending$Gender == "Male") + (spending$City == "Durban")*3, 5))*1000, 0) spending |> plot_ly(x = ~City, y = ~Weekly_Spending, color = ~Gender, type = 'box', colors = c("#A71930", "#0F204B"), width = fig_w, height = fig_h*1.2) |> layout(boxmode = "group") } boxplot_plotly() ``` ## What about business? We could simulate a pair of $VARIMA_2(1,1,1)-tGARCH(1,1)$ financial time series like so: ```{r} varima_plotly <- \() { set.seed(121) n <- 120 e <- mvtnorm::rmvt(n, sigma = c(1, 0.5, 0.5, 1) |> matrix(2), df = 4) x1 <- x2 <- v1 <- v2 <- rep(1, n) for (i in 2:n) { v1[i] <- 0.5 + 0.2*e[i-1, 1]^2 + 0.3*v1[i-1] v2[i] <- 0.5 + 0.2*e[i-1, 2]^2 + 0.3*v2[i-1] e[i,] <- e[i,] * sqrt(c(v1[i], v2[i])) x1[i] <- 0.1 + 0.3*x1[i-1] + 0.3*e[i-1, 1] + e[i, 1] x2[i] <- 0.12 + 0.35*x2[i-1] + 0.2*e[i-1, 2] + e[i, 2] } d <- data.frame( Month = seq_len(n-20), ABC = cumsum(x1[21:n]/60) |> exp() + 30, DEF = cumsum(x2[21:n]/50) |> exp() + 29.5 ) d |> plot_ly(width = fig_w, height = fig_h) |> add_trace(x = ~Month, y = ~ABC, type = 'scatter', mode = 'lines', name = "ABC") |> add_trace(x = ~Month, y = ~DEF, type = 'scatter', mode = 'lines', name = "DEF") |> layout(yaxis = list(title = "Price")) } varima_plotly() ``` ## Trying to get your grandparents to take their pills ```{r} create_predictions_by_age_3d_plotly <- \() { predictions_by_age <- readRDS("predictions_by_age_jointmodel.rds") Ages <- seq(40,70) KUMparsims <- "KUMJointParameterSims.rds" |> readRDS() KM_plot_data <- KUMparsims$KMCurve |> mutate( Treatment = Treatment |> str_remove("^.*="), Time = Time/30 ) treatment_plot_lines <- unique(KM_plot_data$Treatment) gap <- 0.02 endpoint <- 12 time_seq <- seq(0, endpoint, gap) est1 <- predictions_by_age |> sapply(\(d) { d$Estimate[d$Treatment == treatment_plot_lines[1]]} ) |> t() est1l <- predictions_by_age |> sapply(\(d) { d$Cred_Lower[d$Treatment == treatment_plot_lines[1]]} ) |> t() est1u <- predictions_by_age |> sapply(\(d) { d$Cred_Upper[d$Treatment == treatment_plot_lines[1]]} ) |> t() est2 <- predictions_by_age |> sapply(\(d) { d$Estimate[d$Treatment == treatment_plot_lines[2]]} ) |> t() est2l <- predictions_by_age |> sapply(\(d) { d$Cred_Lower[d$Treatment == treatment_plot_lines[2]]} ) |> t() est2u <- predictions_by_age |> sapply(\(d) { d$Cred_Upper[d$Treatment == treatment_plot_lines[2]]} ) |> t() m <- 0.76 plot_ly(x = time_seq, y = Ages, z = est1, type = 'surface', width = fig_w*0.9, height = fig_h*1.5, colorscale = "Blues", opacity = 1, name = treatment_plot_lines[1], cmin = m, cmax = 1, colorbar = list(title = list(text = treatment_plot_lines[1]), nticks = 8)) |> layout(scene = list( xaxis = list(title = "Month", range = c(0, 12)), yaxis = list(title = "Age", range = c(40,70)), zaxis = list(title = "Adherence", range = c(m,1)) )) |> add_surface(z = est1l, colorscale = "Blues", showscale = FALSE, opacity = 0.3, name = treatment_plot_lines[1], cmin = m, cmax = 1) |> add_surface(z = est1u, colorscale = "Blues", showscale = FALSE, opacity = 0.3, name = treatment_plot_lines[1], cmin = m, cmax = 1) |> add_surface(z = est2, colorscale = "Reds", opacity = 1, cmin = m, cmax = 1, reversescale = TRUE, name = treatment_plot_lines[2], colorbar = list(title = list(text = treatment_plot_lines[2]))) |> add_surface(z = est2l, colorscale = "Reds", showscale = FALSE, opacity = 0.3, name = treatment_plot_lines[2], cmin = m, cmax = 1, reversescale = TRUE) |> add_surface(z = est2u, colorscale = "Reds", showscale = FALSE, opacity = 0.3, name = treatment_plot_lines[2], cmin = m, cmax = 1, reversescale = TRUE) } create_predictions_by_age_3d_plotly() ``` ## Pollution at a dam ```{r} #| warning: false crazyts_plot <- \() { crazyts <- readRDS('crazyts.rds') pnttype <- rep('Observed', nrow(crazyts$m_data)) pnttype[crazyts$m_data$IsLeftCensored | crazyts$m_data$IsRightCensored] <- 'Censored' pnttype <- factor(pnttype, levels = rev(unique(pnttype))) cols <- viridisLite::magma(6)[c(2,4)] |> setNames(NULL) plot_ly(width = fig_w, height = fig_h) |> add_trace(x = ~Date, y = ~LogValue, color = ~Location, colors = cols, type = 'scatter', mode = 'markers', symbol = ~pnttype, data = crazyts$m_data) |> add_trace(x = ~Date, y= ~Value, color = ~Location, linetype = ~Line, type = 'scatter', mode = 'lines', colors = cols, data = crazyts$mu_data, visible = "legendonly") } crazyts_plot() ``` ## Bacterial growth on boerewors $$g(t)=a e^{-b e^{-ct}}+\varepsilon_t,\ f(\varepsilon)=\frac{\Gamma(0.5(\nu+1))}{\sqrt{\pi\nu}\Gamma(0.5\nu)}\left(1+\frac{\varepsilon^2}{\nu}\right)^{-0.5(\nu+1)}$$ ```{r} curves_plot_Gompertz <- \() { plt <- readRDS('presentation plot curves1.rds') plt$d$Putrescine <- plt$d$H04 plt$d |> plot_ly(x = ~Hour, y= ~Putrescine, width = fig_w*0.8, height = fig_h*1.2, color = ~StrainDescription, colors = viridisLite::plasma(6), type = 'scatter', mode = 'lines', legendgroup = ~StrainDescription ) |> add_trace(x = ~hour, y = ~value, color = ~StrainDescription, type = 'scatter', mode = 'lines', legendgroup = ~StrainDescription, linetype = ~StrainDescription, data = plt$plot_data) } curves_plot_Gompertz() ``` # Careers - Career options: Actuary, data scientist, statistician, economist, marketing strategist, banker, risk analyst, ... - Employers: Banks, insurance, investment companies, clinical research organisations, marketing companies, ... - UFS options: Actuarial Science, Mathematical Statistics + economics or psychology, Statistics + accounting or biology, Data Science (programming + mathematical statistics), Risk analysis (not the same as risk management), Microbiology + statistics, ... > Begin with the end goal in mind and do what it takes to get there. ## Be real though - Actuarial Science is difficult, much more difficult than medicine - It takes a minimum of 8 years to become an actuary: 4 to 6 years of studying at university and another 4 to 20 years of studying while working - Only a tiny fraction of people make it all the way. - There are great alternatives though: **people starting in actuarial science or something close to it almost always end up getting a good degree, a good honours degree, and rewarding work!** - Some are actuaries, some are data scientists, some are statisticians or statistical programmers, some are in business, marketing, banking, *etc.* ## Why the University of the Free State? - We have many actuaries available to you in person, more than any other university in SA! - All students who finish any honours degrees I've mentioned end up well employed! - A productive study environment - A decades long track record of success # "I've forgotten what you just said." **Just remember one of these:** [www.ufs.ac.za/mathstats](https://www.ufs.ac.za/mathstats) [www.ufs.ac.za/actuarial](https://www.ufs.ac.za/actuarial) ```{r} include_graphics("Mathematical Statistics and Actuarial Science Logo.svg") ```